Changeset 1883

Timestamp:

2001-01-31T14:10:26+13:00 (23 years ago)

Author:

paynter

Message:

Supports new parameters of suffix program and new stopword file locations.
Filtering based on language now works. print_usage updated.

File:

• trunk/gsdl/perllib/classify/phind.pm (modified) (10 diffs)

trunk/gsdl/perllib/classify/phind.pm

@@ -30 +30 @@
 #
 # options are:
-#   button=Name           The label for the classifiers button in the
+#  -button Name           The label for the classifiers button in the
 #                         navigation bar (defaults to "Phrase").
 #  -title Title           The metadata field used to describe each document
@@ -40 +40 @@
 #  -untidy                Do not clean up intermediate files
 #  -suffixmode num        Mode of suffix program (0 = all phrases, 1 = stopword)
-#  -suffixsize num        Number of symbols available to suffix program
 #  -savephrases filename  If set, phrase infomation will be stored in filename
 #                         as text. (By defualt, it is not set.)
@@ -91 +90 @@
 
   options:
-   -title        Title to use on web pages
-   -text
-   -title
-   -button
-   -language
-   -savephrases
-   -suffixsize
-   -suffixmode
-   -thesaurus
-   -untidy
-";
-}
+   -text Fields    The text used to build the phrase hierarchy.
+                   (default: 'section:Title,section:text')
+
+   -title Title    The metadata field used to describe each document.
+                   (default: 'Title')
+
+   -button Name    The label for the classifier screen and button in
+                   navigation bar. 
+                   (default: 'Phrase')
+
+   -language Regex Language or languages to use building hierarchy.
+                   Languages are identified by two-letter country codes
+                   like en (English), es (Spanish), and fr (French).
+                   Language is a regular expression, so 'en|fr' (English or
+                   French) and '..' (match any language) are valid.
+                   (default: 'en'.)
+
+   -savephrases File If set, the phrase infomation will be stored in 
+                     the given file as text. It is probably a good idea 
+                     to use an absolute path. 
+                     (defualt: not set)
+
+   -suffixmode N   The smode parameter to the phrase extraction program.  A
+                   value of 0 means that stopwords are ignored, and of 1
+                   means that stopwords are used.
+                   (default: 1)
+
+   -thesaurus Name Name of a thesaurus stored in phind format in the
+                   collection's etc directory.
+                   (default: not set)
+
+   -untidy         Don't remove working files.
+
+"; }
 
 # Create a new phind browser based on collect.cfg
@@ -174 +195 @@
              q^builddir/.*/^, \$builddir,
              q^savephrases/\d/0^, \$self->{'savephrases'},
-             q^suffixsize/\d+/100000^, \$self->{'suffixsize'},
              q^suffixmode/\d/1^, \$self->{'suffixmode'},
              q^thesaurus/.*/^, \$self->{'thesaurus'},
@@ -250 +270 @@
     print "process: $title\n" if ($verbosity > 2);
 
-    # only consider english-language files
+    # Only consider the file if it is in the correct language
     my $doclanguage = $doc_obj->get_metadata_element ($top_section, "Language");
     my $phrlanguage = $self->{'language_exp'};
@@ -340 +360 @@
     my $verbosity = $self->{'verbosity'};
     my $out = $self->{'outhandle'};
-
     my $phinddir = $self->{'phinddir'};
-    my $language = "english";
-    
+
     if ($verbosity) {
     print $out "\n*** phind.pm generating indexes for ", $self->{'indexes'}, "\n";
@@ -350 +368 @@
     # Construct phind indexes
     my $suffixmode = $self->{'suffixmode'};
-    my $suffixsize = $self->{'suffixsize'};
     my ($command, $status);
     
@@ -360 +377 @@
     # Use the suffix program to generate the phind/phrases file
     print $out "\nExtracting phrases from processed text (with suffix)\n" if $verbosity;
-    &execute("suffix $phinddir $suffixsize $suffixmode", $verbosity, $out);
+    &execute("suffix $phinddir $suffixmode $verbosity", $verbosity, $out);
 
     # Create the phrase file and put phrase numbers in phind/phrases
@@ -552 +569 @@
     my $out = $self->{'outhandle'};
 
-    my $language = "english"; # $self->{'language'};
-
     my $collectiondir = $self->{'collectiondir'};
-
     my $phinddir = $self->{'phinddir'};
+
+    my $language_exp = $self->{'language_exp'};
 
     my ($w, $l, $line, $word);
@@ -576 +592 @@
 
     # Read and store the stopwords
-    my $words = `find $ENV{'GSDLHOME'}/etc/phind/$language -name "*.sw" | xargs cat`;
+    my $stopdir = &util::filename_cat($ENV{'GSDLHOME'}, "etc", "stopwords");
+    my $stopword_files = ();
+    my ($language, $language_dir, $file, $file_name);
     my %stopwords;
-    foreach $w (split(/\s+/, $words)) {
-    $l = lc($w);
-    $stopwords{$l} = $w;
-    }
-    
+
+    # Examine each directory in the stopword directory
+    opendir(STOPDIR, $stopdir);
+    foreach $language (readdir STOPDIR) {
+
+    # Ignore entries that do not match the classifier's language
+    next unless ($language =~ /$language_exp/);
+    $language_dir = &util::filename_cat($stopdir, $language);
+    next unless (-d "$language_dir");
+
+    opendir(LANGDIR, $language_dir);
+    foreach $file (readdir LANGDIR) {
+
+        # Ignore entries that are not stopword files
+        next unless ($file =~ /sw$/);
+        $file_name = &util::filename_cat($language_dir, $file);
+        next unless (-f "$file_name");
+
+        # Read the stopwords
+        open(STOPFILE, "<$file_name");
+        while (<STOPFILE>) {
+        s/^\s+//;
+        s/\s.*//;
+        $word = $_;
+        $l = lc($word);
+        $stopwords{$l} = $word;
+        }
+        close STOPFILE;
+
+    }
+    }
+
     # Read thesaurus information
     if ($thesaurus) {