Index: trunk/indexers/mg/src/text/gs3_mg_passes.c
===================================================================
--- trunk/indexers/mg/src/text/gs3_mg_passes.c	(revision 7431)
+++ trunk/indexers/mg/src/text/gs3_mg_passes.c	(revision 7431)
@@ -0,0 +1,696 @@
+/**************************************************************************
+ *
+ * mg_passes.c -- Driver for the various passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+/* this needs to come first */
+#include "sysfuncs.h"
+
+#ifdef HAVE_MALLINFO
+# include <malloc.h> 
+#endif
+
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+
+#include "longlong.h"
+#include "stemmer.h"
+
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+
+#include "words.h"
+
+/*
+   $Log$
+   Revision 1.1  2004/05/25 03:30:12  kjdon
+   new mg passes for gs3. I thought I had commited this already
+
+   Revision 1.2  2004/04/25 23:01:18  kjdon
+   added a new -M option to mg_passes, allowing maxnumeric to be altered - made this change to keep gsdl3 mg inline with gsdl2 mg.
+
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.3  2001/09/21 12:46:42  kjm18
+   updated mg to be in line with mg_1.3f. Now uses long long for some variables
+   to enable indexing of very large collections.
+
+   Revision 1.2  2001/06/12 23:23:42  jrm21
+   fixed a bug where mg_passes segfaults when trying to print the usage message.
+
+   Revision 1.1  1999/08/10 21:18:12  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:53  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:47  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:13  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:57  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:52  tes
+   * For version 1.1
+   *
+
+*/
+
+static char *RCSID = "$Id$";
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+#define	TERMRECORD	'\002'
+
+unsigned long buf_size = 3 * 1024 * 1024;	/* 3Mb */
+unsigned long invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+unsigned long ChunkLimit = 0;
+char InvfLevel = 2;
+char SkipSGML = 0;
+char MakeWeights = 0;
+FILE *Comp_Stats = NULL;
+int comp_stat_point = 0;
+mg_ullong bytes_processed = 0;
+mg_ullong bytes_received = 0;
+int stemmer_num = 0; /* default to the lovin stemmer */
+int stem_method = 0;
+FILE * Trace;
+char * filename;
+unsigned long num_docs = 0;
+unsigned long block_bytes = 0;
+
+static char Passes = 0;
+static unsigned long trace = 0;
+static int Dump = 0;
+static char **files = NULL;
+static int num_files = 0;
+static char *trace_name = NULL;
+
+typedef struct pass_data
+  {
+    char *name;
+    int (*init) (char *);
+    int (*process) (u_char *, int);
+    int (*done) (char *);
+#ifdef HAVE_TIMES
+    clock_t init_time;
+    clock_t process_time;
+    clock_t done_time;
+#else
+    struct timeval init_time;
+    struct timeval process_time;
+    struct timeval done_time;
+#endif
+  }
+pass_data;
+
+#ifdef HAVE_TIMES
+#define NULL_TIMES 0, 0, 0
+#else
+#define NULL_TIMES {0, 0}, {0, 0}, {0, 0}
+#endif
+
+static pass_data PassData[MAX_PASSES] =
+{
+  {"special", init_special, process_special, done_special, NULL_TIMES},
+  {"text.pass1", init_text_1, process_text_1, done_text_1, NULL_TIMES},
+  {"text.pass2", init_text_2, process_text_2, done_text_2, NULL_TIMES},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1, NULL_TIMES},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2, NULL_TIMES},
+};
+
+static char *usage_str = "\nUSAGE:\n"
+"  %s [-h] [-G] [-D] [-1|-2|-3] [-T1] [-T2] [-I1] [-I2] [-N1]\n"
+"  %*s [-N2] [-W] [-S] [-b buffer-size] [-d dictionary-directory]\n"
+"  %*s [-t trace-point Mb] [-m invf-memory] [-c chunk-limit]\n"
+"  %*s [-n trace-name] [-C comp-stat-size] [-s stem_method]\n"
+"  %*s [-a stemmer] [-M max-numeric] -f doc-collection-name\n";
+
+
+static void 
+usage (char *err)
+{
+  if (err)
+    Message (err);
+  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
+	   strlen (msg_prefix), "",strlen (msg_prefix), "",
+	   strlen (msg_prefix),"");
+  exit (1);
+}
+
+
+
+
+#if 0
+static char *
+str_comma (unsigned long u)
+{
+  static char buf[20];
+  unsigned long a, b, c, d;
+  a = u / 1000000000;
+  u -= a * 1000000000;
+  b = u / 1000000;
+  u -= b * 1000000;
+  c = u / 1000;
+  u -= c * 1000;
+  d = u;
+
+  if (a)
+    sprintf (buf, "%u,%03u,%03u,%03u", a, b, c, d);
+  else if (b)
+    sprintf (buf, "%u,%03u,%03u", b, c, d);
+  else if (c)
+    sprintf (buf, "%u,%03u", c, d);
+  else
+    sprintf (buf, "%u", d);
+  return (buf);
+}
+#endif
+
+
+
+/*
+  int 
+  open_next_file (int in_fd)
+  {
+  if (in_fd > 0)
+  close (in_fd);
+  if (num_files == 0)
+  return (-1);
+  if ((in_fd = open (files[0], O_RDONLY)) == -1)
+  FatalError (1, "Cannot open %s", files[0]); 
+  files++;
+  num_files--;
+  return (in_fd);
+  }
+*/
+
+void clear_variables() {
+
+buf_size = 3 * 1024 * 1024;	/* 3Mb */
+invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+ChunkLimit = 0;
+InvfLevel = 2;
+SkipSGML = 0;
+MakeWeights = 0;
+Comp_Stats = NULL;
+comp_stat_point = 0;
+bytes_processed = 0;
+bytes_received = 0;
+stemmer_num = 0; /* default to the lovin stemmer */
+stem_method = 0;
+Trace = NULL;
+filename = NULL;
+num_docs = 0;
+block_bytes = 0;
+
+Passes = 0;
+trace = 0;
+Dump = 0;
+files = NULL;
+num_files = 0;
+trace_name = NULL;
+
+
+}
+void set_invf_level(char level) {
+
+  switch (level) {
+  case '1':
+    InvfLevel = 1;
+    break;
+  case '2':
+    InvfLevel = 2;
+    break;
+  case '3':
+    InvfLevel = 3;
+    break;
+  }
+
+}
+void set_stem_options(char * stemmer, int method) {
+  stemmer_num = stemmernumber (stemmer);
+  printf("stemmer num set to %d\n", stemmer_num);
+  stem_method = method & STEMMER_MASK;
+
+}
+
+void set_paths(char * basep, char* filen) {
+  int len = strlen(filen);
+  if (filename) {
+    Xfree (filename);
+    filename = NULL;
+  }
+
+  filename = Xstrdup (filen);
+  set_basepath(basep);
+  // put this here for now
+  SkipSGML=0;
+  Dump=1;
+  trace = 512;
+  if (!trace_name)
+    trace_name = make_name (filename, TRACE_SUFFIX, NULL);
+  if (!(Trace = fopen (trace_name, "a")))
+    Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
+  else
+    setbuf (Trace, NULL);
+  
+  
+}
+
+void add_pass (char pass_type, char pass_num) {
+
+  switch(pass_type) {
+  case 'S':
+    Passes |= SPECIAL;
+    break;
+  case 'I':
+  case 'N':
+    if (pass_num == '1')
+      Passes |= IVF_PASS_1;
+    else if (pass_num == '2')
+      Passes |= IVF_PASS_2;
+    else
+      fprintf(stderr, "Invalid pass number %c for pass type %c\n", pass_num, pass_type);
+    break;
+  case 'T':
+    if (pass_num == '1')
+      Passes |= TEXT_PASS_1;
+    else if (pass_num == '2')
+      Passes |= TEXT_PASS_2;
+    else
+      fprintf(stderr, "Invalid pass number %c for pass type %c\n", pass_num, pass_type);
+    break;
+  }
+
+}
+ProgTime StartTime, InitTime, ProcTime, DoneTime;
+
+void 
+init_driver ()
+{
+  int pass;
+
+  GetTime (&StartTime);
+  
+  for (pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass)) {
+      pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+      struct tms tims;
+      times (&tims);
+      pd->init_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+      struct rusage ru;
+      
+      getrusage (RUSAGE_SELF, &ru);
+      pd->init_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+      pd->init_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+      if (pd->init (filename) == COMPERROR)
+	FatalError (1, "Error during init of \"%s\"", pd->name);
+      
+#ifdef HAVE_TIMES
+      times (&tims);
+      pd->init_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+      getrusage (RUSAGE_SELF, &ru);
+      pd->init_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+      pd->init_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+      time_normalise (&pd->init_time);
+#endif
+    }
+  }
+  GetTime (&InitTime);
+}
+
+
+void process_document(u_char *buffer, int len) {
+  int pass;
+  bytes_processed += len;
+
+  printf("process doc, len=%d\n",len);
+#ifndef QUIET
+  if (!len)
+    Message ("Warning : Processing zero length document");
+#endif
+  
+  for (pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass))
+      {
+	register pass_data *pd = &PassData[pass];
+	
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->process_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+	register struct timeval *tv = &pd->process_time;
+	
+	getrusage (RUSAGE_SELF, &ru);
+	tv->tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	tv->tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->process ((u_char *) buffer, len) == COMPERROR)
+	  {
+	    Message ("Error during processing of \"%s\"", pd->name);
+	    if (Dump || Trace)
+	      {
+		int i;
+		FILE *f = Trace ? Trace : stderr;
+		fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+		for (i = 0; i < len; i++)
+		  {
+		    char ch = buffer[i];
+		    if (ch == '\1' || ch == '\2')
+		      ch = '\n';
+		    putc (ch, f);
+		  }
+		fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+	      }
+	    if (Trace)
+	      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+		       bytes_processed, num_docs,
+		       ElapsedTime (&StartTime, NULL));
+	    exit (1);
+	  }
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->process_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	tv->tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	tv->tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+      }
+  }
+  num_docs++;
+  if (Trace)
+    {
+      block_bytes += len;
+      if (block_bytes >= trace)
+	{
+#ifdef HAVE_MALLINFO
+	  struct mallinfo mi;
+	  mi = mallinfo ();
+	  block_bytes -= trace;
+	  fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs |%7.3f Mb | %s\n",
+		   bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+		   ElapsedTime (&StartTime, NULL));
+#else
+	  block_bytes -= trace;
+	  fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+		   bytes_processed, num_docs,
+		   ElapsedTime (&StartTime, NULL));
+#endif
+	}
+    }
+}
+
+void finalise_driver() {
+  int pass;
+#ifndef HAVE_TIMES
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      time_normalise (&PassData[pass].process_time);
+#endif
+
+  GetTime (&ProcTime);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->done_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+	
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->done (filename) == COMPERROR)
+	  FatalError (1, "Error during done of \"%s\"", pd->name);
+
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->done_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+	time_normalise (&pd->done_time);
+#endif
+      }
+  if (Trace)
+    {
+#ifdef HAVE_MALLINFO
+      struct mallinfo mi;
+      mi = mallinfo ();
+      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs |%7.3f Mb | %s\n",
+	       bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+	       ElapsedTime (&StartTime, NULL));
+#else
+      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+	       bytes_processed, num_docs,
+	       ElapsedTime (&StartTime, NULL));
+#endif
+    }
+
+  GetTime (&DoneTime);
+
+  Message ("");
+  Message ("%10s      :    init         process        done", "");
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+	char it[15], pt[15], dt[15];
+#ifdef HAVE_TIMES
+	strcpy (it, cputime_string (pd->init_time));
+	strcpy (pt, cputime_string (pd->process_time));
+	strcpy (dt, cputime_string (pd->done_time));
+#else
+	strcpy (it, cputime_string (&pd->init_time));
+	strcpy (pt, cputime_string (&pd->process_time));
+	strcpy (dt, cputime_string (&pd->done_time));
+#endif
+	Message ("%-10s      : %s   %s   %s", pd->name, it, pt, dt);
+      }
+  Message ("");
+  Message ("Init time       : %s", ElapsedTime (&StartTime, &InitTime));
+  Message ("Process time    : %s", ElapsedTime (&InitTime, &ProcTime));
+  Message ("Done time       : %s", ElapsedTime (&ProcTime, &DoneTime));
+  Message ("Total time      : %s", ElapsedTime (&StartTime, &DoneTime));
+  Message ("Documents       : %u", num_docs);
+  Message ("Bytes received  : %" ULL_FS, bytes_received);
+  Message ("Bytes processed : %" ULL_FS, bytes_processed);
+  Message ("Process Rate    : %.1f kB per cpu second",
+   (double) bytes_processed / (ProcTime.CPUTime - InitTime.CPUTime) / 1024);
+  //free (buffer);
+}
+
+
+
+int main (int argc, char **argv)
+{
+  int ch, in_fd;
+ 
+  msg_prefix = argv[0];
+
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "hC:WGSD123f:d:b:T:I:t:m:N:c:n:s:a:M:")) != -1)
+    {
+      switch (ch)
+	{
+	case 'G':
+	  SkipSGML = 1;
+	  break;
+	case 'S':
+	  Passes |= SPECIAL;
+	  break;
+	case '1':
+	  InvfLevel = 1;
+	  break;
+	case '2':
+	  InvfLevel = 2;
+	  break;
+	case '3':
+	  InvfLevel = 3;
+	  break;
+	case 'f':
+	  filename = optarg;
+	  break;
+	case 'n':
+	  trace_name = optarg;
+	  break;
+	case 'D':
+	  Dump = 1;
+	  break;
+	case 'W':
+	  MakeWeights = 1;
+	  break;
+	case 'd':
+	  set_basepath (optarg);
+	  break;
+	case 'a':
+	  stemmer_num = stemmernumber (optarg);
+	  break;
+	case 's':
+	  stem_method = atoi (optarg) & STEMMER_MASK;
+	  break;
+	case 'b':
+	  buf_size = atoi (optarg) * 1024;
+	  break;
+	case 'C':
+	  comp_stat_point = atoi (optarg) * 1024;
+	  break;
+	case 'c':
+	  ChunkLimit = atoi (optarg);
+	  break;
+	case 'm':
+	  invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'I':
+	case 'N': /* N kept for compatability */
+	  if (*optarg == '1')
+	    Passes |= IVF_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= IVF_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 'T':
+	  if (*optarg == '1')
+	    Passes |= TEXT_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= TEXT_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 't':
+	  trace = (unsigned long) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'M':
+	  SetEnv ("maxnumeric", optarg, NULL);
+	  break;
+	case 'h':
+	case '?':
+	  usage (NULL);
+	}
+    }
+
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if (buf_size < MIN_BUF)
+    FatalError (1, "The buffer size must exceed 1024 bytes.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (optind < argc)
+    {
+      if ((in_fd = open (argv[optind], O_RDONLY)) == -1)
+	FatalError (1, "Cannot open %s", argv[optind]);
+      files = &argv[optind + 1];
+      num_files = argc - (optind + 1);
+    }
+  else
+    in_fd = 0;			/* stdin */
+
+
+  if (trace)
+    {
+      if (!trace_name)
+	trace_name = make_name (filename, TRACE_SUFFIX, NULL);
+      if (!(Trace = fopen (trace_name, "a")))
+	Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
+      else
+	setbuf (Trace, NULL);
+    }
+  else
+    Trace = NULL;
+
+  if (comp_stat_point)
+    {
+      char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
+      if (!(Comp_Stats = fopen (name, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	Message ("Unable to open \"%s\". No comp. stats. will be generated.",
+		 name);
+    }
+
+
+  if (Trace)
+    {
+      int i;
+      fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (i = 0; i < argc; i++)
+	fprintf (Trace, "%s ", argv[i]);
+      fprintf (Trace, "\n\n");
+    }
+
+  init_driver ();
+  char * doc = "hello there kath";
+  process_document(doc, 17);
+  finalise_driver();
+  if (Trace)
+    fclose (Trace);
+
+  if (Comp_Stats)
+    fclose (Comp_Stats);
+
+  return 0;
+}
