Changeset 7460 for trunk/gsdl3/src/packages/mg/src

Timestamp:

2004-05-27T13:50:24+12:00 (20 years ago)

Author:

kjdon

Message:

adding more options to mgpasses, tidying up

File:

• trunk/gsdl3/src/packages/mg/src/text/mg_passes_4jni.c (modified) (9 diffs)

trunk/gsdl3/src/packages/mg/src/text/mg_passes_4jni.c

@@ -1 +1 @@
 /**************************************************************************
  *
- * mg_passes.c -- Driver for the various passes
+ * mg_passes_4jni.c -- Driver for the various passes
  * Copyright (C) 1994  Neil Sharman
  *
@@ -35 +35 @@
 #include "stemmer.h"
 
-
 #include "mg_files.h"
 #include "mg.h"
@@ -113 +112 @@
 };
 
-static char *usage_str = "\nUSAGE:\n"
-"  %s [-h] [-G] [-D] [-1|-2|-3] [-T1] [-T2] [-I1] [-I2] [-N1]\n"
-"  %*s [-N2] [-W] [-S] [-b buffer-size] [-d dictionary-directory]\n"
-"  %*s [-t trace-point Mb] [-m invf-memory] [-c chunk-limit]\n"
-"  %*s [-n trace-name] [-C comp-stat-size] [-s stem_method]\n"
-"  %*s [-a stemmer] [-M max-numeric] -f doc-collection-name\n";
-
-
-static void 
-usage (char *err)
-{
-  if (err)
-    Message (err);
-  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
-       strlen (msg_prefix), "",strlen (msg_prefix), "",
-       strlen (msg_prefix),"");
-  exit (1);
-}
-
-
-
-
-#if 0
-static char *
-str_comma (unsigned long u)
-{
-  static char buf[20];
-  unsigned long a, b, c, d;
-  a = u / 1000000000;
-  u -= a * 1000000000;
-  b = u / 1000000;
-  u -= b * 1000000;
-  c = u / 1000;
-  u -= c * 1000;
-  d = u;
-
-  if (a)
-    sprintf (buf, "%u,%03u,%03u,%03u", a, b, c, d);
-  else if (b)
-    sprintf (buf, "%u,%03u,%03u", b, c, d);
-  else if (c)
-    sprintf (buf, "%u,%03u", c, d);
-  else
-    sprintf (buf, "%u", d);
-  return (buf);
-}
-#endif
-
-
-
-/*
-  int 
-  open_next_file (int in_fd)
-  {
-  if (in_fd > 0)
-  close (in_fd);
-  if (num_files == 0)
-  return (-1);
-  if ((in_fd = open (files[0], O_RDONLY)) == -1)
-  FatalError (1, "Cannot open %s", files[0]); 
-  files++;
-  num_files--;
-  return (in_fd);
-  }
-*/
-
+
+/* clear all the settings from one mg_passes run to the next */
 void clear_variables() {
 
@@ -207 +142 @@
   
 }
-void set_invf_level(char level) {
-
-  switch (level) {
-  case '1':
-    InvfLevel = 1;
-    break;
-  case '2':
-    InvfLevel = 2;
-    break;
-  case '3':
-    InvfLevel = 3;
-    break;
-  }
-
-}
-void set_inversion_limit(int limit) {
-  invf_buffer_size = limit * 1024 * 1024;
-}
-
-void ignore_sgml_tags(int ignore) {
-  if (ignore) {
-    SkipSGML = 1;
-  } else {
-    SkipSGML = 0;
-  }
-}
-
-void set_buffer_size(long size) {
-  buf_size = size * 1024;
-  if (buf_size <  MIN_BUF) {
-    buf_size = MIN_BUF;
-  }
-}
-
-void set_stem_options(char * stemmer, int method) {
-  stemmer_num = stemmernumber (stemmer);
-  stem_method = method & STEMMER_MASK;
-
-}
-
-void set_filename(char * filen) {
-  int len = strlen(filen);
-  if (filename) {
-    Xfree (filename);
-    filename = NULL;
-  }
-  filename = Xstrdup (filen);
-  // put this here for now
-  Dump=1;
-  trace = 512;
-  if (!trace_name)
-    trace_name = make_name (filename, TRACE_SUFFIX, NULL);
-  if (!(Trace = fopen (trace_name, "a")))
-    Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
-  else
-    setbuf (Trace, NULL);
-  
-}
-
-
+
+/* ################################################## */
+/* the following are methods to set all the variables that used to be
+   set by command line args */
+
+/* -S, -T1, -T2, -I1, -I2, args to mg_passes */
 void add_pass (char pass_type, char pass_num) {
 
@@ -291 +172 @@
     break;
   }
-
-}
+}
+
+/* -D arg to mg_passes */
+void dump_failed_document(int dump) {
+  Dump = dump;
+}
+
+/* -G arg to mg_passes */
+void ignore_sgml_tags(int ignore) {
+  if (ignore) {
+    SkipSGML = 1;
+  } else {
+    SkipSGML = 0;
+  }
+}
+
+/* -b arg to mg_passes */
+void set_buffer_size(long size) {
+  buf_size = size * 1024;
+  if (buf_size <  MIN_BUF) {
+    buf_size = MIN_BUF;
+  }
+}
+
+/* -c arg to mg_passes */
+void set_chunk_limit(long chunk_limit) {
+  ChunkLimit = chunk_limit;
+}
+
+/* -C arg to mg_passes */
+void set_comp_stat_point(int stat_point) {
+  comp_stat_point = stat_point * 1024;
+}
+
+/* -f arg to mg_passes */
+void set_filename(char * filen) {
+  int len = strlen(filen);
+  if (filename) {
+    Xfree (filename);
+    filename = NULL;
+  }
+  filename = Xstrdup (filen);  
+}
+
+/* -m arg to mg_passes */
+void set_inversion_limit(int limit) {
+  invf_buffer_size = limit * 1024 * 1024;
+}
+
+/* -1, -2, -3 args to mg_passes */
+void set_invf_level(char level) {
+  switch (level) {
+  case '1':
+    InvfLevel = 1;
+    break;
+  case '2':
+    InvfLevel = 2;
+    break;
+  case '3':
+    InvfLevel = 3;
+    break;
+  }
+}
+
+/* -W arg to mg_passes */
+void set_make_weights(int make_w) {
+  MakeWeights = make_w;
+}
+
+/* -M arg to mg_passes */
+void set_max_numeric(int max_numeric) {
+  SetEnv ("maxnumeric", max_numeric, NULL);
+}
+
+/* -a, -s args to mg_passes */
+void set_stem_options(char * stemmer, int method) {
+  stemmer_num = stemmernumber (stemmer);
+  stem_method = method & STEMMER_MASK;
+}
+
+/* -t arg to mg_passes */
+void set_trace_point(int tracepos) {
+  trace = (unsigned long) (tracepos * 1024 * 1024);
+}
+
+/* -n arg to mg_passes */
+void set_trace_file(char * filen) {
+  int len = strlen(filen);
+  if (trace_name) {
+    Xfree (trace_name);
+    trace_name = NULL;
+  }
+  trace_name = Xstrdup (filen);
+}
+
+/* ############################################### */
+/* The old driver method has been split into 3:
+init_driver, process_document (called numdocs times), 
+finalise_driver.
+The above set vars methods should all be called before init_driver.
+*/
+
+
 ProgTime StartTime, InitTime, ProcTime, DoneTime;
 
@@ -298 +280 @@
 init_driver ()
 {
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (trace)
+    {
+      if (!trace_name)
+    trace_name = make_name (filename, TRACE_SUFFIX, NULL);
+      if (!(Trace = fopen (trace_name, "a")))
+    Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
+      else
+    setbuf (Trace, NULL);
+    }
+  else
+    Trace = NULL;
+
+  if (comp_stat_point)
+    {
+      char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
+      if (!(Comp_Stats = fopen (name, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    Message ("Unable to open \"%s\". No comp. stats. will be generated.",
+         name);
+    }
+
+  if (Trace)
+    {
+      int i;
+      fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      /* print out the args to mg_passes */
+      fprintf (Trace, "\n\n");
+    }
+
   int pass;
 
@@ -338 +360 @@
   bytes_processed += len;
 
-  printf("process doc, len=%d\n",len);
 #ifndef QUIET
   if (!len)
@@ -501 +522 @@
    (double) bytes_processed / (ProcTime.CPUTime - InitTime.CPUTime) / 1024);
   //free (buffer);
-}
-
-
-
-int main (int argc, char **argv)
-{
-  int ch, in_fd;
- 
-  msg_prefix = argv[0];
-
-  opterr = 0;
-  while ((ch = getopt (argc, argv, "hC:WGSD123f:d:b:T:I:t:m:N:c:n:s:a:M:")) != -1)
-    {
-      switch (ch)
-    {
-    case 'G':
-      SkipSGML = 1;
-      break;
-    case 'S':
-      Passes |= SPECIAL;
-      break;
-    case '1':
-      InvfLevel = 1;
-      break;
-    case '2':
-      InvfLevel = 2;
-      break;
-    case '3':
-      InvfLevel = 3;
-      break;
-    case 'f':
-      filename = optarg;
-      break;
-    case 'n':
-      trace_name = optarg;
-      break;
-    case 'D':
-      Dump = 1;
-      break;
-    case 'W':
-      MakeWeights = 1;
-      break;
-    case 'd':
-      set_basepath (optarg);
-      break;
-    case 'a':
-      stemmer_num = stemmernumber (optarg);
-      break;
-    case 's':
-      stem_method = atoi (optarg) & STEMMER_MASK;
-      break;
-    case 'b':
-      buf_size = atoi (optarg) * 1024;
-      break;
-    case 'C':
-      comp_stat_point = atoi (optarg) * 1024;
-      break;
-    case 'c':
-      ChunkLimit = atoi (optarg);
-      break;
-    case 'm':
-      invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
-      break;
-    case 'I':
-    case 'N': /* N kept for compatability */
-      if (*optarg == '1')
-        Passes |= IVF_PASS_1;
-      else if (*optarg == '2')
-        Passes |= IVF_PASS_2;
-      else
-        usage ("Invalid pass number");
-      break;
-    case 'T':
-      if (*optarg == '1')
-        Passes |= TEXT_PASS_1;
-      else if (*optarg == '2')
-        Passes |= TEXT_PASS_2;
-      else
-        usage ("Invalid pass number");
-      break;
-    case 't':
-      trace = (unsigned long) (atof (optarg) * 1024 * 1024);
-      break;
-    case 'M':
-      SetEnv ("maxnumeric", optarg, NULL);
-      break;
-    case 'h':
-    case '?':
-      usage (NULL);
-    }
-    }
-
-  if (!filename || *filename == '\0')
-    FatalError (1, "A document collection name must be specified.");
-
-  if (buf_size < MIN_BUF)
-    FatalError (1, "The buffer size must exceed 1024 bytes.");
-
-  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
-    FatalError (1, "I1 and I2 cannot be done simultaneously.");
-
-  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
-    FatalError (1, "T1 and T2 cannot be done simultaneously.");
-
-  if (!Passes)
-    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
-
-  if (optind < argc)
-    {
-      if ((in_fd = open (argv[optind], O_RDONLY)) == -1)
-    FatalError (1, "Cannot open %s", argv[optind]);
-      files = &argv[optind + 1];
-      num_files = argc - (optind + 1);
-    }
-  else
-    in_fd = 0;          /* stdin */
-
-
-  if (trace)
-    {
-      if (!trace_name)
-    trace_name = make_name (filename, TRACE_SUFFIX, NULL);
-      if (!(Trace = fopen (trace_name, "a")))
-    Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
-      else
-    setbuf (Trace, NULL);
-    }
-  else
-    Trace = NULL;
-
-  if (comp_stat_point)
-    {
-      char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
-      if (!(Comp_Stats = fopen (name, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
-    Message ("Unable to open \"%s\". No comp. stats. will be generated.",
-         name);
-    }
-
-
-  if (Trace)
-    {
-      int i;
-      fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
-      for (i = 0; i < argc; i++)
-    fprintf (Trace, "%s ", argv[i]);
-      fprintf (Trace, "\n\n");
-    }
-
-  init_driver ();
-  /* here we have to do something to process docs from stdin */
-  finalise_driver();
+
   if (Trace)
     fclose (Trace);
@@ -658 +529 @@
     fclose (Comp_Stats);
 
-  return 0;
-}
+}
+