Changeset 11994 for trunk/gsdl/perllib/classify.pm

Timestamp:

2006-07-04T15:06:34+12:00 (18 years ago)

Author:

davidb

Message:

Improved support for incremental addition: instead of having to run the
classifier pass of buildcol.pl from scratch (i.e. read in all documents
from the archives folder) so correct browse structures are formed -- a
simple to implement strategy, but not very efficient -- the first layer
of a classifier structure is now reconstructed from the GDBM file. Then
the new files in the archives directory are added, and then finally the
completed browser structure is formed.

File:

• trunk/gsdl/perllib/classify.pm (modified) (2 diffs)

trunk/gsdl/perllib/classify.pm

@@ -31 +31 @@
 require AllList;
 use gsprintf;
+
+use GDBM_File;
 
 
@@ -139 +141 @@
     }
 }
+
+
+
+# takes a hashref containing the metadata for a gdbmfile entry, and extracts 
+# the childrens numbers (from the 'contains' entry).    
+# assumes format is ".1;".2;".3
+sub get_children {  
+    my ($doc_db_hash) = @_; 
+
+    my $children = undef;
+
+    $childs = $doc_db_hash->{'contains'};
+    if (defined ($childs)) {
+    $childs =~ s/\@$//;  #remove trailing @
+    $childs =~ s/^\"\.//; #remove initial ".
+    @$children = split /\;\"\./, $childs;
+    
+    }
+
+    return $children;
+}
+
+    
+sub recurse_sections {
+    my ($doc_obj, $children, $parentoid, $parentsection, $gdbm_recs) = @_;
+
+    return if (!defined $children);
+
+    foreach my $child (sort { $a <=> $b} @$children) {
+    $doc_obj->create_named_section("$parentsection.$child");
+    my $doc_db_rec = $gdbm_recs->{"$parentoid.$child"};
+    my $doc_db_hash = db_rec_to_hash($doc_db_rec);
+
+    # get child's children
+    my $newchildren = &get_children($doc_db_hash); 
+
+    # add content for current section
+    add_section_content($doc_obj, "$parentsection.$child", $doc_db_hash);
+
+    # process all the children if there are any
+    if (defined ($newchildren))
+    {
+        recurse_sections($doc_obj, $newchildren, "$parentoid.$child", 
+                 "$parentsection.$child", $gdbm_recs);
+    }
+    }
+}                        
+
+
+sub add_section_content {
+    my ($doc_obj, $cursection, $doc_db_hash) = @_;
+ 
+    foreach $key (keys %$doc_db_hash) {
+    #don't need to store these metadata
+    next if $key =~ /(thistype|childtype|contains|docnum|doctype|classifytype)/i;
+    # but do want things like hastxt and archivedir
+    my @items = split /@/, $doc_db_hash->{$key};
+    map {$doc_obj->add_metadata ($cursection, $key, $_); } @items;
+
+    }
+}
+
+
+# gets all the metadata from a gdbm file entry, and puts it into a hashref
+sub db_rec_to_hash {
+    
+    my ($gdb_str_ref) = @_;
+
+    my $hashref = {};
+
+    my @entries = split(/\n/, $gdb_str_ref);
+    foreach $entry (@entries) {
+    my($key, $value) = ($entry =~ /^<([^>]*)>(.*?)$/ );
+    $hashref->{$key} .= '@' if defined $hashref->{$key};
+    $hashref->{$key} .= $value;
+    
+    }
+   
+    return $hashref;
+}                     
+
+
+sub reconstruct_doc_objs_metadata
+{
+    my ($fulldbname) = @_;
+
+    tie %gdbm_recs, 'GDBM_File', $fulldbname, &GDBM_WRCREAT, 0640;
+
+    # dig out top level doc sections
+    my %top_sections = ();
+    foreach my $key ( keys %gdbm_recs )
+    {
+    my $md_rec = $gdbm_recs{$key};
+    my $md_hash = db_rec_to_hash($md_rec);
+
+    if ((defined $md_hash->{'doctype'}) && ($md_hash->{'doctype'} eq "doc")) {
+        next if ($key =~ m/\./);
+        $top_sections{$key} = $md_hash;
+    }
+    }
+
+    # for greenstone document objects based on metadata in gdbm file
+    my @all_docs = ();
+    foreach my $oid ( keys %top_sections )
+    {
+    my $doc_db_hash = $top_sections{$oid};
+
+    my $doc_obj = new doc();
+    $doc_obj->set_OID($oid);
+
+    my $top = $doc_obj->get_top_section();
+        add_section_content ($doc_obj, $top, $doc_db_hash);
+        my $children = &get_children($doc_db_hash);
+        recurse_sections($doc_obj, $children, $oid, $top, \%gdbm_recs);
+
+    push(@all_docs,$doc_obj);
+    }    
+
+    untie %gdbm_recs;
+
+    return \@all_docs;   
+}
+
+
+
+
 
 # classify_doc lets each of the classifiers classify a document