Changeset 8362 for trunk/gsdl/src/phind/generate

Timestamp:

2004-10-18T15:30:40+13:00 (20 years ago)

Author:

kjdon

Message:

added a new option to the phind classifier: min_occurs. this is the minimum phrase frequency needed to be included in the hierarchy

Location:

trunk/gsdl/src/phind/generate

Files:

• phrase.cpp (modified) (5 diffs)
• suffix.cpp (modified) (5 diffs)
• suffix.h (modified) (1 diff)

trunk/gsdl/src/phind/generate/phrase.cpp

@@ -46 +46 @@
 
 // Phrase constructor functions
-
 
 Phrase::Phrase(symbol *words, cellcount size, int direction) {
@@ -659 +658 @@
 
     // If the expansion occurs more than once and is not delimited, expand it
-    if ((*(next.back) > LASTDELIMITER) && (next.suffixFrequency >= 2)) {
+    if ((*(next.back) > LASTDELIMITER) && (next.suffixFrequency >= minOccurs)) {
       next.expandWhileUniqueSuffixExtension();
       results.push_back(next);
@@ -683 +682 @@
 
     // If the expansion occurs more than once and is not delimited, expand it
-    if ((*(next.forward) > LASTDELIMITER) && (next.prefixFrequency >= 2)) {
+    if ((*(next.forward) > LASTDELIMITER) && (next.prefixFrequency >= minOccurs)) {
       next.expandWhileUniquePrefixExtension();
       results.push_back(next);
@@ -751 +750 @@
 
   // if the phrase occurs only once, do nothing
-  if (suffixFrequency < 2)
+  if (suffixFrequency < minOccurs)
     return 0;
 
@@ -794 +793 @@
 
   // if the phrase occurs only once, do nothing
-  if (prefixFrequency < 2)
+  if (prefixFrequency < minOccurs)
     return 0;
 

trunk/gsdl/src/phind/generate/suffix.cpp

@@ -72 +72 @@
 // Do we accept any phrase, or do we eliminate those ending with stopwords ?
 int phraseMode = ANYPHRASE; //STOPWORDS;
-
+// What is the minimum phrase frequency for a phrase to be included in the hierarchy
+int minOccurs = 2;
 
 // The filestem of the collection's phindex directory
@@ -718 +719 @@
   // Command-line arguments
   // argv[1] is the phindex directory
-  // argv[2] is the maximum array symbol length (optional) 
-  // argv[3] is the mode, where 1 is stopword mode (optional) 
-  if (argc < 2) {
-    cerr << "Usage: " << argv[0] << " phind-directory mode [verbosity]" << endl;
+  // argv[2] is the mode, where 1 is stopword mode 
+  // argv[3] is the min_occurs, - minimum occurrence frequency for a phrase to be included in the hierarchy
+  // argv[4] is opitonal verbosity
+  if (argc < 4) {
+    cerr << "Usage: " << argv[0] << " phind-directory mode min-phrase-freq [verbosity]" << endl;
     exit(1);
   }
@@ -732 +734 @@
   assert((phraseMode == STOPWORDS) || (phraseMode == ANYPHRASE));
 
+  minOccurs = atoi(argv[3]);
+  assert((minOccurs > 0));
+
   // optional verbosity parameter
-  if (argc == 4) {
-    verbosity = atoi(argv[3]);
+  if (argc == 5) {
+    verbosity = atoi(argv[4]);
     assert (verbosity >= 0);
   }
@@ -892 +897 @@
     // The system used to work like this; it is easy to implement, but
     // it explodes the size of the indexes.  So: would it be useful?  
-    if (p.suffixFrequency > 1) {
+    if (p.suffixFrequency >= minOccurs) {
       // Get minimal expansions of the phrase
       getExpansions(p, result);
@@ -1000 +1005 @@
 
       // Ignore the phrase if it only occurs once
-      if (p.suffixFrequency < 2)
+      if (p.suffixFrequency < minOccurs)
     continue;
 

trunk/gsdl/src/phind/generate/suffix.h

@@ -68 +68 @@
 // Are we allowed to terminate a phrase on a stopword?
 extern int phraseMode;
+extern int minOccurs;
 
 #define ANYPHRASE 0