Changeset 8362

Timestamp:

2004-10-18T15:30:40+13:00 (20 years ago)

Author:

kjdon

Message:

added a new option to the phind classifier: min_occurs. this is the minimum phrase frequency needed to be included in the hierarchy

Location:

trunk/gsdl

Files:

• perllib/classify/Phind.pm (modified) (6 diffs)
• perllib/strings.rb (modified) (1 diff)
• src/phind/generate/phrase.cpp (modified) (5 diffs)
• src/phind/generate/suffix.cpp (modified) (5 diffs)
• src/phind/generate/suffix.h (modified) (1 diff)

trunk/gsdl/perllib/classify/Phind.pm

@@ -113 +113 @@
     'type' => "int",
     'deft' => "1",
+    'range' => "0,1",
+    'reqd' => "no" },
+      { 'name' => "min_occurs",
+    'desc' => "{Phind.min_occurs}",
+    'type' => "int",
+    'deft' => "2",
+    'range' => "1,",
     'reqd' => "no" },
       { 'name' => "thesaurus",
@@ -176 +183 @@
              q^savephrases/.*/^, \$self->{'savephrases'},
              q^suffixmode/\d/1^, \$self->{'suffixmode'},
+             q^min_occurs/\d/2^, \$self->{'min_occurs'},
              q^thesaurus/.*/^, \$self->{'thesaurus'},
              q^untidy^, \$self->{'untidy'},
@@ -288 +296 @@
     print $txthandle "$doclimit\n";
 
-    # iterarate over the required indexes and store their text
+    # iterate over the required indexes and store their text
     my $indexes = $self->{'indexes'};
     my $text = "";
@@ -379 +387 @@
     # Construct phind indexes
     my $suffixmode = $self->{'suffixmode'};
+    my $min_occurs = $self->{'min_occurs'};
     my ($command, $status);
     
@@ -390 +399 @@
     print $out "\nExtracting phrases from processed text (with suffix)\n" if $verbosity;
     print STDERR "<Phase name='ExtractingPhrase'/>\n" if $gli;
-    &execute("suffix \"$phinddir\" $suffixmode $verbosity", $verbosity, $out);
+    &execute("suffix \"$phinddir\" $suffixmode $min_occurs $verbosity", $verbosity, $out);
 
     # check that we generated some files. It's not necessarily an error if
@@ -1399 +1408 @@
 #
 # Read phrases.3 and restore vocabulary information. Then write 
-# this data to the MGPP input files (pwrod.txt and pdata.txt) and
+# this data to the MGPP input files (pword.txt and pdata.txt) and
 # (if requested) to the saved phrases file.
 

trunk/gsdl/perllib/strings.rb

@@ -368 +368 @@
 Phind.language:Language or languages to use building hierarchy. Languages are identified by two-letter country codes like en (English), es (Spanish), and fr (French). Language is a regular expression, so 'en|fr' (English or French) and '..' (match any language) are valid.
 
+Phind.min_occurs:The minimum number of times a phrase must appear in the text to be included in the phrase hierarchy.
+
 Phind.savephrases:If set, the phrase infomation will be stored in the given file as text. It is probably a good idea to use an absolute path.
 

trunk/gsdl/src/phind/generate/phrase.cpp

@@ -46 +46 @@
 
 // Phrase constructor functions
-
 
 Phrase::Phrase(symbol *words, cellcount size, int direction) {
@@ -659 +658 @@
 
     // If the expansion occurs more than once and is not delimited, expand it
-    if ((*(next.back) > LASTDELIMITER) && (next.suffixFrequency >= 2)) {
+    if ((*(next.back) > LASTDELIMITER) && (next.suffixFrequency >= minOccurs)) {
       next.expandWhileUniqueSuffixExtension();
       results.push_back(next);
@@ -683 +682 @@
 
     // If the expansion occurs more than once and is not delimited, expand it
-    if ((*(next.forward) > LASTDELIMITER) && (next.prefixFrequency >= 2)) {
+    if ((*(next.forward) > LASTDELIMITER) && (next.prefixFrequency >= minOccurs)) {
       next.expandWhileUniquePrefixExtension();
       results.push_back(next);
@@ -751 +750 @@
 
   // if the phrase occurs only once, do nothing
-  if (suffixFrequency < 2)
+  if (suffixFrequency < minOccurs)
     return 0;
 
@@ -794 +793 @@
 
   // if the phrase occurs only once, do nothing
-  if (prefixFrequency < 2)
+  if (prefixFrequency < minOccurs)
     return 0;
 

trunk/gsdl/src/phind/generate/suffix.cpp

@@ -72 +72 @@
 // Do we accept any phrase, or do we eliminate those ending with stopwords ?
 int phraseMode = ANYPHRASE; //STOPWORDS;
-
+// What is the minimum phrase frequency for a phrase to be included in the hierarchy
+int minOccurs = 2;
 
 // The filestem of the collection's phindex directory
@@ -718 +719 @@
   // Command-line arguments
   // argv[1] is the phindex directory
-  // argv[2] is the maximum array symbol length (optional) 
-  // argv[3] is the mode, where 1 is stopword mode (optional) 
-  if (argc < 2) {
-    cerr << "Usage: " << argv[0] << " phind-directory mode [verbosity]" << endl;
+  // argv[2] is the mode, where 1 is stopword mode 
+  // argv[3] is the min_occurs, - minimum occurrence frequency for a phrase to be included in the hierarchy
+  // argv[4] is opitonal verbosity
+  if (argc < 4) {
+    cerr << "Usage: " << argv[0] << " phind-directory mode min-phrase-freq [verbosity]" << endl;
     exit(1);
   }
@@ -732 +734 @@
   assert((phraseMode == STOPWORDS) || (phraseMode == ANYPHRASE));
 
+  minOccurs = atoi(argv[3]);
+  assert((minOccurs > 0));
+
   // optional verbosity parameter
-  if (argc == 4) {
-    verbosity = atoi(argv[3]);
+  if (argc == 5) {
+    verbosity = atoi(argv[4]);
     assert (verbosity >= 0);
   }
@@ -892 +897 @@
     // The system used to work like this; it is easy to implement, but
     // it explodes the size of the indexes.  So: would it be useful?  
-    if (p.suffixFrequency > 1) {
+    if (p.suffixFrequency >= minOccurs) {
       // Get minimal expansions of the phrase
       getExpansions(p, result);
@@ -1000 +1005 @@
 
       // Ignore the phrase if it only occurs once
-      if (p.suffixFrequency < 2)
+      if (p.suffixFrequency < minOccurs)
     continue;
 

trunk/gsdl/src/phind/generate/suffix.h

@@ -68 +68 @@
 // Are we allowed to terminate a phrase on a stopword?
 extern int phraseMode;
+extern int minOccurs;
 
 #define ANYPHRASE 0