Changeset 1604 for trunk/gsdl/src/phind/generate/phproc.pm

Timestamp:

2000-10-17T12:35:59+13:00 (24 years ago)

Author:

paynter

Message:

Numerous improvements for use with the new phindcgi script. The main ones
are that three MGPP databases are now created (document data, phrase data,
and word search) and that the data extracted from each document is set
explicitly in the collection configuration file (usually it will be
something like document:text or section:Title).

File:

• trunk/gsdl/src/phind/generate/phproc.pm (modified) (3 diffs)

trunk/gsdl/src/phind/generate/phproc.pm

@@ -38 +38 @@
 
 sub new {
-    my ($class, $archive_dir, $phindex_dir, 
+    my ($class, $archive_dir, $phindex_dir, $phindcfg,
     $language, $delimiter, $verbosity, $outhandle) = @_;
 
     my $self = new docproc ();
     
-    # $self->{'collection'} = $collection;
     $self->{'archive_dir'} = $archive_dir;
     $self->{'phindex_dir'} = $phindex_dir;
+    $self->{'indexes'} = $phindcfg;
 
     $language =~ s/,/\|/g;
@@ -60 +60 @@
     $self->{'txthandle'} = TEXT;
 
-    &util::rm("$phindex_dir/mg-d.txt") if (-e "$phindex_dir/mg-d.txt");
-    open(DOCS, ">$phindex_dir/mg-d.txt")
-    || die "Cannot open $phindex_dir/mg-d.txt: $!";
+    my $docfile = &util::filename_cat("$phindex_dir", "docs.txt");
+    &util::rm($docfile) if (-e $docfile);
+    open(DOCS, ">$docfile")
+    || die "Cannot open $docfile: $!";
     $self->{'dochandle'} = DOCS;
 
@@ -95 +96 @@
     my $dochandle = $self->{'dochandle'};
     # print "dochandle: =$dochandle=\n";
-    print $dochandle "$OID\t$title\n";
-    
-    # store the text
-    $text = convert_gml_to_tokens($doc_obj->get_text());
-
+    print $dochandle "<Document>\t$OID\t$title\n";
+    
+    # XXX
+    # Store the text of this object
+    my $indexlist = $self->{'indexes'};
+    my @parts;
+    my ($index, $part, $level, $field, $section, $data, $text);
+
+    # Output the document delimiter
     my $txthandle = $self->{'txthandle'};
-    print $txthandle $self->{'delimiter'}, "\n$text\n";
+    print $txthandle $self->{'delimiter'}, "\n";
+
+    # Iterarate over all the indexes specified in collect.cfg and
+    # add their text to the clauses file.
+    foreach $index (@$indexlist) {
+    $text = "";
+
+    # Iterate over all the feilds in each index
+    @parts = split(/,/, $index);
+    foreach $part (@parts) {
+
+        # Each field has a level and a data element ((e.g. document:Title)
+        ($level, $field) = split(/:/, $part);
+        die unless ($level && $field);
+
+        # Extract the text from every section
+        # (In phind, document:text and section:text are equivalent)
+        if ($field eq "text") {
+        $data = "";
+        $section = $doc_obj->get_top_section();
+        while (defined($section)) {
+            $data .= $doc_obj->get_text($section) . "\n";
+            $section = $doc_obj->get_next_section($section);
+        }
+        $text .= convert_gml_to_tokens($data) . "\n";
+        }
+
+        # Extract a metadata field from a document
+        elsif ($level eq "document") {
+        $data = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $field);
+        $text .= convert_gml_to_tokens($data) . "\n";
+        } 
+        
+        # Extract metadata from every section in a document
+        elsif ($level eq "section") {
+        $data = "";
+        $section = $doc_obj->get_top_section();
+        while (defined($section)) {
+            $data .= $doc_obj->get_metadata_element($section, $field) . "\n";
+            $section = $doc_obj->get_next_section($section);
+        }
+        $text .= convert_gml_to_tokens($data) . "\n";
+        } 
+        
+        # Some sort of specification which I don't understand
+        else {
+        die "Unknown level ($level) in phind key ($part) in phind index ($index)\n";
+        }
+
+    }
+
+    # print the text
+    print $txthandle "$text";
+
+    }
 }