Index: /trunk/gsdl/src/phind/generate/Makefile.in
===================================================================
--- /trunk/gsdl/src/phind/generate/Makefile.in	(revision 2866)
+++ /trunk/gsdl/src/phind/generate/Makefile.in	(revision 2867)
@@ -43,5 +43,5 @@
 
 
-HEADERS = suffix.h phrase.h
+HEADERS = suffix.h phrase.h check.h
 SOURCES = suffix.cpp phrase.cpp
 OBJECTS = phrase.o suffix.o
@@ -65,2 +65,4 @@
 suffix : $(OBJECTS)
 	$(CXX) $(LDFLAGS) -o $@ $(OBJECTS)
+
+
Index: /trunk/gsdl/src/phind/generate/check.h
===================================================================
--- /trunk/gsdl/src/phind/generate/check.h	(revision 2867)
+++ /trunk/gsdl/src/phind/generate/check.h	(revision 2867)
@@ -0,0 +1,153 @@
+// check.h -- manipulate the suffixCheck array
+//
+// The suffixCheck array is used to prevent a phrase being added to
+// the results of getExpansions.  It works by declaring an array that
+// has one bit for each entry in the suffix array, which is cleared at
+// the start of getExpansions.  As getExpansions adds each phrase to
+// the result set, we set the bits in the suffixCheck array that
+// correspond to the positions of of the phrase in the suffixArray.
+
+// (The point is that getExpansions can later check whether a phrase
+// or its prefix has been seen before by looking up the bit
+// corresponding to its first occurence in the suffixCheck array: if
+// the bit is set, a subphrase of this phrase is already in the
+// results already, and the current phrase can be ignored.)
+
+#include "suffix.h"
+
+// The suffixCheck array
+typedef unsigned char check;
+check *suffixCheck;
+
+// The length of the suffixCheck array
+cellcount checkLength;
+
+// Some useful bitmask constants 
+const check ALLBITS = 255;
+const check LEFTMOSTBIT = (ALLBITS << 7);
+const check LEFTMOSTBITS[] = {0, 
+			      (ALLBITS << 7), (ALLBITS << 6), (ALLBITS << 5), (ALLBITS << 4),
+			      (ALLBITS << 3), (ALLBITS << 2), (ALLBITS << 1),  ALLBITS};
+//const check LEFTMOSTBIT = 128;
+//const check LEFTMOSTBITS[] = {0, 128, 192, 224, 240, 248, 252, 254, 255}; 
+
+
+// Allocate the initial memory for suffixCheck
+inline void allocateSuffixCheck(cellcount number_of_symbols) 
+{
+  checkLength = number_of_symbols / (sizeof(check) * 8) + 1;
+  suffixCheck = new check[checkLength];
+  if (suffixCheck == NULL) {
+    cerr << "Suffix error: out of memory allocating suffixCheck array" << endl; 
+    exit(2);
+  }
+  memset(suffixCheck, 0, sizeof(check) * checkLength);
+}
+
+
+// Set all bits in suffixCheck to 0
+inline void clearSuffixCheck() 
+{
+  memset(suffixCheck, 0, sizeof(check) * checkLength);
+}
+
+
+// Get the value of a particular bit in suffixCheck
+inline int getSuffixCheck(cellindex suff) 
+{
+  cellindex cell = suff >> 3;
+  check remainder = suff & 0x07; // the last 3 bits
+  if (suffixCheck[cell] & (LEFTMOSTBIT >> remainder)) {
+    return 1;
+  }
+  return 0;
+}                                                                               
+
+
+// Set the value of a bit in suffixCheck to 1
+inline void setSuffixCheck(cellindex suff) 
+{
+  cellindex cell = suff >> 3;
+  check remainder = suff & 0x07; // the last 3 bits
+  suffixCheck[cell] |= (LEFTMOSTBIT >> remainder);
+}  
+
+
+// Set the value of a range of bits in suffixCheck to 1
+void setSuffixCheck(cellindex first, cellindex last) 
+{
+  for (cellcount i = first; i <= last; ++i)
+    setSuffixCheck(i);
+}
+
+
+// Set the bits in a cell in the suffixArray.
+// Bits are indexed 0-7.
+inline void setCellBits(cellindex cell_offset, unsigned first_bit, unsigned last_bit) 
+{
+  unsigned number_of_bits = last_bit - first_bit + 1;
+  check mask = (LEFTMOSTBITS[number_of_bits] >> first_bit);
+  suffixCheck[cell_offset] |= mask;
+}
+
+
+inline void setSuffixCheck_new(cellindex first, cellindex last) 
+{
+  // If only one bit is set, use simpler function
+  // We should be able to remove this case.
+  if (first == last) {
+    setSuffixCheck(first);
+    return;
+  }
+
+  // Find the first and last cells in which bits are set
+  cellindex first_cell = first >> 3;
+  cellindex last_cell = last >> 3;
+
+  // If all the values are in the same cell, set them
+  if (first_cell == last_cell) {
+    setCellBits(first_cell, (first & 0x07), (last & 0x07));
+    return;
+  }
+
+  // Set the bits in the first and last cells
+  setCellBits(first_cell, (first & 0x07), 7);
+  setCellBits(last_cell, 0, (last & 0x07));
+
+  // Set the bits in the intermediate cells
+  ++first_cell;
+  if(last_cell > first_cell)
+    memset(suffixCheck + first_cell, ALLBITS, 
+	   (last_cell - first_cell) * sizeof(check));
+}
+
+
+// Print the suffixCheck array (for debugging)
+void printSuffixCheck() 
+{
+  for (cellcount i = 0; i < checkLength; ++i) {
+    cout << "cell " << i << "  \t";
+    cout << "(" << (i * 8) << "-" << (i * 8 + 7) << ")    \t";
+    for (cellindex j = 0; j < 8; ++j) {
+      cout << getSuffixCheck(i * 8 + j);
+      // cout << " (" << (i * 8 + j) << ") "; 
+    }
+    cout << " (" << (unsigned int) suffixCheck[i] << ")\n";
+  }
+}
+
+
+//  To test, put this at the start of main
+
+//  allocateSuffixCheck(300);
+//  clearSuffixCheck();
+//  setSuffixCheck(20);
+//  setSuffixCheck(35,50);
+//  setSuffixCheck(68,150);
+//  setSuffixCheck(170,174);
+//  setSuffixCheck(220,230);
+//  setSuffixCheck(247);
+//  setSuffixCheck(256);
+//  printSuffixCheck();
+//  exit(0);
+
Index: /trunk/gsdl/src/phind/generate/suffix.cpp
===================================================================
--- /trunk/gsdl/src/phind/generate/suffix.cpp	(revision 2866)
+++ /trunk/gsdl/src/phind/generate/suffix.cpp	(revision 2867)
@@ -2,18 +2,18 @@
  *
  * Suffix.cpp -- Extract the repeated phrases in the input with suffix
- *                and prefix arrays (cgn & gwp's simpler algorithm,
- *                 and kjm's improvements).
+ *               and prefix arrays (cgn & gwp's simpler algorithm,
+ *               and kjm's improvements).
  *
  * Copyright 2000 Gordon W. Paynter
  * Copyright 2000 The New Zealand Digital Library Project
  *
- * A component of the Greenstone digital library software
- * from the New Zealand Digital Library Project at the
- * University of Waikato, New Zealand.
+ * A component of the Greenstone digital library software from the 
+ * New Zealand Digital Library Project at the University of Waikato, 
+ * New Zealand.
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
@@ -56,4 +56,5 @@
 #include "suffix.h"
 #include "phrase.h"
+#include "check.h"
 
 // Global variables declared in suffix.h
@@ -63,8 +64,4 @@
 symbol  **suffixArray;
 symbol  **prefixArray;
-check    *suffixCheck;
-
-// the length of the check array
-cellcount checkLength;
 
 // How many documents are in this collection?
@@ -93,7 +90,4 @@
 int pointerCompare(const void *, const void *);
 
-// some bit manipulation functions for the check arrays, defined below
-int getSuffixCheck(cellindex suff);
-void setSuffixCheck(cellindex suff);
                                            
 // Functions for implementing "phrase memory".  These let us "remember"
@@ -142,6 +136,6 @@
   // Initialise the candidates, check array, and various variables.
   sort(candidates.begin(), candidates.end(), isShorter);
-  memset(suffixCheck, 0, sizeof(check)*checkLength);
   unsigned minimum_length = candidates.begin()->length;
+  clearSuffixCheck();
   
   // Try to add each candidate to the results set, ignoring the non-minimal
@@ -153,8 +147,7 @@
     bool shorter_found = false;
     
-    // Check for shorter and shorter versions of the tenporary phrase
+    // Check for shorter and shorter versions of the temporary phrase
     while (temp_phrase.length >= minimum_length && !shorter_found) {
       temp_phrase.ensureSuffixFound();
-      //if (suffixCheck[temp_phrase.firstSuffixIndex] == 0)
       if (getSuffixCheck(temp_phrase.firstSuffixIndex)==0)
 	temp_phrase.shortenByOneAtPrefix();
@@ -166,10 +159,9 @@
     }
       
+    // If no shorter phrase is found, use this one
     if (!shorter_found) {
       results.push_back(*candidate);
       candidate->ensureSuffixFound();
-      for (cellcount k = candidate->firstSuffixIndex; k <= candidate->lastSuffixIndex; ++k)
-	//suffixCheck[k] = candidate->length;
-	setSuffixCheck(k);
+      setSuffixCheck(candidate->firstSuffixIndex, candidate->lastSuffixIndex);
     }
   }
@@ -254,18 +246,4 @@
 }
 
-int getSuffixCheck(cellindex suff) {
-  cellindex cell = suff >> 3;
-  check remainder = suff & 0x07; // the last 3 bits
-  if (suffixCheck[cell]& (1 << remainder)) {
-    return 1;
-  }
-  return 0;
-}                                                                               
-void setSuffixCheck(cellindex suff) {
-  cellindex cell = suff >> 3;
-  check remainder = suff & 0x07; // the last 3 bits
-  suffixCheck[cell] |= (1 << remainder);
-
-}  
 
 // Read the clauses.numbers file into the "symbols" array.
@@ -735,4 +713,5 @@
 }
 
+
 int main (int argc, char * argv[]) {
 
@@ -762,5 +741,4 @@
     cout << "suffix: the phrase extraction program" << endl;
   }
-
   if (verbosity > 1) {
     if (phraseMode == STOPWORDS) {
@@ -786,14 +764,17 @@
   else firstContent = firstStopSymbol;
 
+  // Allocate memory for the suffix & prefix arrays
   cellcount contentLength = 0;
   contentLength = getContentCount(firstContent);
-
-  // Create the suffix & prefix arrays
   suffixArray = new symbol *[contentLength];
   prefixArray = new symbol *[contentLength];
-
-  cellcount here=0;
+  if (prefixArray == NULL) {
+    cerr << "Suffix: not enough memory to hold " << inputLength << " symbols." << endl;
+    exit(2);
+  }  
+  allocateSuffixCheck(contentLength);
+
   // Initialise prefix and suffix arrays, only use the needed suffixes
-  for (cellcount j = 0; j < inputLength; j++) {
+  for (cellcount j = 0, here = 0; j < inputLength; j++) {
     if (symbols[j]>=firstContent) {
       suffixArray[here] = &symbols[j];
@@ -805,14 +786,4 @@
   qsort(prefixArray, contentLength, sizeof(symbol *), prefixCompare);
 
-  checkLength = contentLength/8 + 1;
-  suffixCheck = new check[checkLength];
-  if (suffixCheck == NULL) {
-    cerr << "Suffix error: not enough memory to hold " << inputLength << " symbols." << endl;
-    exit(2);
-  }  
-  memset(suffixCheck, 0, sizeof(check)*checkLength);
-
-  cout <<"\ngenerating the phrase hierarchy\n\n";
-  
   // Create the document arrays
   if (verbosity > 1) {
@@ -829,4 +800,8 @@
   // each phrase occurs in.
   documentArray = new symbol *[numberOfDocuments];  
+  if (documentArray == NULL) {
+    cerr << "Suffix: out of memory allocating document arrays." << endl;
+    exit(2);
+  }  
 
   // just scan through the input text to find the doc starts
@@ -857,4 +832,6 @@
   // it and add its expansions to the set of output phrases.
 
+  cout <<"\ngenerating the phrase hierarchy\n\n";
+  
   // Store the phrase data in the phrases file
   char phraseDataName[FILENAME_MAX];
Index: /trunk/gsdl/src/phind/generate/suffix.h
===================================================================
--- /trunk/gsdl/src/phind/generate/suffix.h	(revision 2866)
+++ /trunk/gsdl/src/phind/generate/suffix.h	(revision 2867)
@@ -46,9 +46,4 @@
 typedef unsigned int frequency;
 
-// The check type is used when we want to store low frequency values.
-// Required range: 0 - 8 (could be recoded to use booleans)
-typedef unsigned char check;
-
-
 // Global variables 
 
@@ -62,7 +57,5 @@
 // Suffix and prefix arrays are used to extract phrases
 extern symbol  **suffixArray;
-extern check    *suffixCheck;
 extern symbol  **prefixArray;
-extern check    *prefixCheck;
 
 // Collection-specific information about the first stopword/content symbols
