Changeset 2867 for trunk/gsdl/src

Timestamp:

2001-11-28T15:07:12+13:00 (22 years ago)

Author:

paynter

Message:

Moved all the sufficCheck functionality into the check.h header and
inlined it.

Location:

trunk/gsdl/src/phind/generate

Files:

• Makefile.in (modified) (2 diffs)
• check.h (added)
• suffix.cpp (modified) (14 diffs)
• suffix.h (modified) (2 diffs)

trunk/gsdl/src/phind/generate/Makefile.in

@@ -43 +43 @@
 
 
-HEADERS = suffix.h phrase.h
+HEADERS = suffix.h phrase.h check.h
 SOURCES = suffix.cpp phrase.cpp
 OBJECTS = phrase.o suffix.o
@@ -65 +65 @@
 suffix : $(OBJECTS)
     $(CXX) $(LDFLAGS) -o $@ $(OBJECTS)
+
+

trunk/gsdl/src/phind/generate/suffix.cpp

@@ -2 +2 @@
  *
  * Suffix.cpp -- Extract the repeated phrases in the input with suffix
- *                and prefix arrays (cgn & gwp's simpler algorithm,
- *                 and kjm's improvements).
+ *               and prefix arrays (cgn & gwp's simpler algorithm,
+ *               and kjm's improvements).
  *
  * Copyright 2000 Gordon W. Paynter
  * Copyright 2000 The New Zealand Digital Library Project
  *
- * A component of the Greenstone digital library software
- * from the New Zealand Digital Library Project at the
- * University of Waikato, New Zealand.
+ * A component of the Greenstone digital library software from the 
+ * New Zealand Digital Library Project at the University of Waikato, 
+ * New Zealand.
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
@@ -56 +56 @@
 #include "suffix.h"
 #include "phrase.h"
+#include "check.h"
 
 // Global variables declared in suffix.h
@@ -63 +64 @@
 symbol  **suffixArray;
 symbol  **prefixArray;
-check    *suffixCheck;
-
-// the length of the check array
-cellcount checkLength;
 
 // How many documents are in this collection?
@@ -93 +90 @@
 int pointerCompare(const void *, const void *);
 
-// some bit manipulation functions for the check arrays, defined below
-int getSuffixCheck(cellindex suff);
-void setSuffixCheck(cellindex suff);
                                            
 // Functions for implementing "phrase memory".  These let us "remember"
@@ -142 +136 @@
   // Initialise the candidates, check array, and various variables.
   sort(candidates.begin(), candidates.end(), isShorter);
-  memset(suffixCheck, 0, sizeof(check)*checkLength);
   unsigned minimum_length = candidates.begin()->length;
+  clearSuffixCheck();
   
   // Try to add each candidate to the results set, ignoring the non-minimal
@@ -153 +147 @@
     bool shorter_found = false;
     
-    // Check for shorter and shorter versions of the tenporary phrase
+    // Check for shorter and shorter versions of the temporary phrase
     while (temp_phrase.length >= minimum_length && !shorter_found) {
       temp_phrase.ensureSuffixFound();
-      //if (suffixCheck[temp_phrase.firstSuffixIndex] == 0)
       if (getSuffixCheck(temp_phrase.firstSuffixIndex)==0)
     temp_phrase.shortenByOneAtPrefix();
@@ -166 +159 @@
     }
       
+    // If no shorter phrase is found, use this one
     if (!shorter_found) {
       results.push_back(*candidate);
       candidate->ensureSuffixFound();
-      for (cellcount k = candidate->firstSuffixIndex; k <= candidate->lastSuffixIndex; ++k)
-    //suffixCheck[k] = candidate->length;
-    setSuffixCheck(k);
+      setSuffixCheck(candidate->firstSuffixIndex, candidate->lastSuffixIndex);
     }
   }
@@ -254 +246 @@
 }
 
-int getSuffixCheck(cellindex suff) {
-  cellindex cell = suff >> 3;
-  check remainder = suff & 0x07; // the last 3 bits
-  if (suffixCheck[cell]& (1 << remainder)) {
-    return 1;
-  }
-  return 0;
-}                                                                               
-void setSuffixCheck(cellindex suff) {
-  cellindex cell = suff >> 3;
-  check remainder = suff & 0x07; // the last 3 bits
-  suffixCheck[cell] |= (1 << remainder);
-
-}  
 
 // Read the clauses.numbers file into the "symbols" array.
@@ -735 +713 @@
 }
 
+
 int main (int argc, char * argv[]) {
 
@@ -762 +741 @@
     cout << "suffix: the phrase extraction program" << endl;
   }
-
   if (verbosity > 1) {
     if (phraseMode == STOPWORDS) {
@@ -786 +764 @@
   else firstContent = firstStopSymbol;
 
+  // Allocate memory for the suffix & prefix arrays
   cellcount contentLength = 0;
   contentLength = getContentCount(firstContent);
-
-  // Create the suffix & prefix arrays
   suffixArray = new symbol *[contentLength];
   prefixArray = new symbol *[contentLength];
-
-  cellcount here=0;
+  if (prefixArray == NULL) {
+    cerr << "Suffix: not enough memory to hold " << inputLength << " symbols." << endl;
+    exit(2);
+  }  
+  allocateSuffixCheck(contentLength);
+
   // Initialise prefix and suffix arrays, only use the needed suffixes
-  for (cellcount j = 0; j < inputLength; j++) {
+  for (cellcount j = 0, here = 0; j < inputLength; j++) {
     if (symbols[j]>=firstContent) {
       suffixArray[here] = &symbols[j];
@@ -805 +786 @@
   qsort(prefixArray, contentLength, sizeof(symbol *), prefixCompare);
 
-  checkLength = contentLength/8 + 1;
-  suffixCheck = new check[checkLength];
-  if (suffixCheck == NULL) {
-    cerr << "Suffix error: not enough memory to hold " << inputLength << " symbols." << endl;
-    exit(2);
-  }  
-  memset(suffixCheck, 0, sizeof(check)*checkLength);
-
-  cout <<"\ngenerating the phrase hierarchy\n\n";
-  
   // Create the document arrays
   if (verbosity > 1) {
@@ -829 +800 @@
   // each phrase occurs in.
   documentArray = new symbol *[numberOfDocuments];  
+  if (documentArray == NULL) {
+    cerr << "Suffix: out of memory allocating document arrays." << endl;
+    exit(2);
+  }  
 
   // just scan through the input text to find the doc starts
@@ -857 +832 @@
   // it and add its expansions to the set of output phrases.
 
+  cout <<"\ngenerating the phrase hierarchy\n\n";
+  
   // Store the phrase data in the phrases file
   char phraseDataName[FILENAME_MAX];

trunk/gsdl/src/phind/generate/suffix.h

@@ -46 +46 @@
 typedef unsigned int frequency;
 
-// The check type is used when we want to store low frequency values.
-// Required range: 0 - 8 (could be recoded to use booleans)
-typedef unsigned char check;
-
-
 // Global variables 
 
@@ -62 +57 @@
 // Suffix and prefix arrays are used to extract phrases
 extern symbol  **suffixArray;
-extern check    *suffixCheck;
 extern symbol  **prefixArray;
-extern check    *prefixCheck;
 
 // Collection-specific information about the first stopword/content symbols