Changeset 1882 for trunk/gsdl/src/phind/generate/suffix.cpp

Timestamp:

2001-01-31T14:08:50+13:00 (23 years ago)

Author:

paynter

Message:

The length of the main symbols array is now calculated from the
clauses.numbers file, not passedin as a command-line parameter.
This is a little slower, as we have to make an extra pass over
the text, but oh-so-much-more convienient. Requires changes to
command line arguments (which now support verbosity).

File:

• trunk/gsdl/src/phind/generate/suffix.cpp (modified) (10 diffs)

trunk/gsdl/src/phind/generate/suffix.cpp

@@ -45 +45 @@
 
 // Global variables declared in suffix.h
-cellcount symbol_array_length;
 cellcount inputLength;
 
@@ -69 +68 @@
 int pointerCompare(const void *, const void *);
 
-int readNumbers(symbol *numbers);
+int readNumbers();
 void readStatistics();
 
@@ -103 +102 @@
   // argv[3] is the mode, where 1 is stopword mode (optional) 
   if (argc < 2) {
-    cerr << "Usage: " << argv[0] << " collection-directory [max-array-size [mode]]" << endl;
+    cerr << "Usage: " << argv[0] << " phind-directory mode [verbosity]" << endl;
     exit(1);
   }
@@ -110 +109 @@
   strcpy(collection, argv[1]);
 
-  // Symbol length parameter
-  if (argc >= 3) {
-    symbol_array_length = atol(argv[2]);
-    assert(symbol_array_length);
-  } else {
-    symbol_array_length = 1000;
-  }
-
-  // Stopword mode parameter
+  // mode parameter
+  phraseMode = atoi(argv[2]);
+  assert((phraseMode == STOPWORDS) || (phraseMode == ANYPHRASE));
+
+  // optional verbosity parameter
   if (argc == 4) {
-    phraseMode = atoi(argv[3]);
-    assert(phraseMode == STOPWORDS);
-  }
+    verbosity = atoi(argv[3]);
+    assert (verbosity >= 0);
+  }
+
   if (verbosity) {
+    cout << "Suffix phrase extraction program" << endl;
+  }
+
+  if (verbosity > 1) {
     if (phraseMode == STOPWORDS) {
-      cout << "STOPWORDS mode: no phrase may begin or end with a stopword" << endl;
+      cout << "Stopwords mode: no phrase may begin or end with a stopword" << endl;
     } else {
-      cout << "ALLPHRASE mode: extract every phrase that occurs more than once" << endl;
+      cout << "AllPhrase mode: extract every phrase that occurs more than once" << endl;
     }
   }
@@ -135 +135 @@
 
   // Read the numbers file
-  symbols = new (symbol)[symbol_array_length];
-  readNumbers(symbols);
-
+  readNumbers();
 
   // Create the suffix & prefix arrays
-  if (verbosity) {
-    cout << "Create suffix & prefix arrays for " << inputLength << " symbols" << endl;
-  }
   suffixArray = new (symbol *)[inputLength];
   prefixArray = new (symbol *)[inputLength];
   suffixCheck = new (check)[inputLength];
   prefixCheck = new (check)[inputLength];
+  if (prefixCheck == NULL) {
+    cerr << "Suffix error: not enough memory to hold " << inputLength
+     << " symbols." << endl;
+    exit(2);
+  }  
+
+  // Initialise prefix and suffix arrays
   for (cellcount j = 0; j < inputLength; j++) {
     suffixArray[j] = &symbols[j];
@@ -156 +158 @@
 
   // Create the document arrays
-  if (verbosity) {
-    cout << "Create document arrays for " << numberOfDocuments << " documents" << endl;
-  }
   if (numberOfDocuments == 0) {
     cerr << "There are no documents in this collection!" << endl;
     exit(1);
+  }
+  if (verbosity > 1) {
+    cout << "Allocating document arrays for " << numberOfDocuments << " documents" << endl;
   }
 
@@ -226 +228 @@
 
   // PASS NUMBER 1
-  if (verbosity) {
-    cout << endl<< "suffix: starting pass " << phrasePass << endl;
+  if (verbosity > 1) {
+    cout << "Starting pass " << phrasePass << endl;
   }
 
@@ -348 +350 @@
     phrasePass++;
     if (verbosity) {
-      cout << endl << "Starting pass " << phrasePass << endl;
+      cout << "Starting pass " << phrasePass << endl;
     }
 
@@ -664 +666 @@
 
 
-// Read the numbers file into an array of symbols.
-//
-// Each number is a symbol number; it is essential that the first
-// symbol (and no others) be 0 and the last symbol (and no others) 
-// be 1.
+// Read the clauses.numbers file into the "symbols" array.
+//
+// Each number in the file is a symbol number; it is essential that 
+// the first symbol (and no others) be COLLECTIONSTART and the last
+// symbol (and no others) be COLLECTIONEND.
 //
 // Return the number of numbers in the array.
 
-int readNumbers(symbol *numbers) {
+int readNumbers() {
 
   char filename[FILENAME_MAX];
   sprintf(filename, "%s/clauses.numbers", collection);
   if (verbosity) {
-    cout << "Reading numbers from: " << filename << endl;
+    cout << "Reading numbers file: " << filename << endl;
   }
 
@@ -687 +689 @@
   }
 
+  // Count the number of symbols
+  inputLength = 0;
+  symbol word;
+  while (inFile >> word) {
+    inputLength++;
+  }
+  inFile.close();
+
+  // Allocate the symbbols array
+  if (verbosity > 1) {
+    cout << "Allocating symbol arrays for " << inputLength << " symbols" << endl;
+  }
+  symbols = new (symbol)[inputLength];
+  if (symbols == NULL) {
+    cerr << "Suffix error: not enough memory to hold " << inputLength
+     << " symbols." << endl;
+    exit(2);
+  }
+
   // Read the numbers file into the numbers array
-  symbol word;
-  cellcount length = 0;
+  if (verbosity > 2) {
+    cout << "Reading the numbers" << endl;
+  }
+  inFile.open(filename, ios::in);
+  cellcount next = 0;
   numberOfDocuments = 0;
-  while ((inFile >> word) && (length < symbol_array_length)){
-    numbers[length++] = word;
+  while (inFile >> word) {
+    symbols[next++] = word;
     if (word == DOCUMENTSTART) {
       numberOfDocuments++;
     }
   }
-  
-  // Make sure we were able to read all the numbers
-  if (length >= symbol_array_length) {
-    cerr << "Error: the symbol array is too a short to hold " << filename
-     << endl << "It is currently set to " <<  symbol_array_length
-     << " and can be adjusted at the command line." << endl;
-    exit(1);
-  }
+  inFile.close();
   
   // Make sure the numbers file is intact
-  assert(numbers[0] == COLLECTIONSTART);
-  assert(numbers[length-1] == COLLECTIONEND);
-
-  // Record the length of the Input file
-  inputLength = length;
-
-  return length;
+  assert(symbols[0] == COLLECTIONSTART);
+  assert(symbols[next-1] == COLLECTIONEND);
+
+  return inputLength;
 }