Changeset 14934

Timestamp:

2007-12-20T21:53:14+13:00 (16 years ago)

Author:

davidb

Message:

Changes to allow statistic calculations for metadata coverage, i.e. for this docment which metadata set prefixes are used, which fields within those prefixes are used, and how many times. This is then agregated over the all documents and the summary stored as collection level metadata.

Location:

gsdl/trunk/perllib

Files:

• basebuilder.pm (modified) (3 diffs)
• basebuildproc.pm (modified) (3 diffs)
• lucenebuildproc.pm (modified) (1 diff)
• mgppbuilder.pm (modified) (2 diffs)

gsdl/trunk/perllib/basebuilder.pm

@@ -368 +368 @@
     $self->{'buildproc'}->zero_reset(); 
 
+    $self->{'buildproc'}->{'mdprefix_fields'} = {};
+
     if ($self->{'keepold'}) {
     # create flat classify structure, ready for new docs to be added
@@ -377 +379 @@
 
    
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+           "", {}, $self->{'buildproc'}, $self->{'maxdocs'},0, $self->{'gli'});
+
     # this has changed to only output collection meta if its 
     # not in the config file
     $self->output_collection_meta($handle);
-    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
-           "", {}, $self->{'buildproc'}, $self->{'maxdocs'},0, $self->{'gli'});
     
     # output classification information
@@ -562 +565 @@
 }
 
-# default is to output an empty [collection] entry
+
+
+sub output_collection_meta_start {
+    my $self = shift(@_);
+    my ($handle) = @_;
+    
+    print $handle "[collection]\n";
+
+}  
+
+sub output_collection_meta_sets {
+    my $self = shift(@_);
+    my ($handle) = @_;
+
+    my $mdprefix_fields = $self->{'buildproc'}->{'mdprefix_fields'};
+
+    foreach my $prefix (keys %$mdprefix_fields)
+    {
+    print $handle "<metadataset>$prefix\n";
+
+    foreach my $field (keys %{$mdprefix_fields->{$prefix}})
+    {
+        my $val = $mdprefix_fields->{$prefix}->{$field};
+
+        print $handle "<metadatalist-$prefix>$field\n";
+        print $handle "<metadatafreq-$prefix-$field>$val\n";
+    }
+
+    }
+
+}  
+
+sub output_collection_meta_end {
+    my $self = shift(@_);
+    my ($handle) = @_;
+    
+    print $handle ('-' x 70) . "\n";;
+
+}  
+
+
+# default is to output the metadata sets (prefixes) used in collection
+
 sub output_collection_meta {
     my $self = shift(@_);
     my ($handle) = @_;
-    
-    print $handle "[collection]\n". ('-' x 70) . "\n";;
+
+    $self->output_collection_meta_start($handle);
+    $self->output_collection_meta_sets($handle);
+    $self->output_collection_meta_end($handle);
 
 }  

gsdl/trunk/perllib/basebuildproc.pm

@@ -297 +297 @@
 }
 
+
+
+sub infodb_metadata_stats
+{
+    my $self = shift (@_);
+    my ($field) = @_;
+
+    # Keep some statistics relating to metadata sets used and
+    # frequency of particular metadata fields within each set
+
+    # Union of metadata prefixes and frequency of fields
+    # (both scoped for this document alone, and across whole collection)
+    
+    if ($field =~ m/^(.+)\.(.*)$/) {
+    my $prefix = $1;
+    my $core_field = $2;
+
+    $self->{'doc_mdprefix_fields'}->{$prefix}->{$core_field}++;
+    $self->{'mdprefix_fields'}->{$prefix}->{$core_field}++;
+    }
+    elsif ($field =~ m/^[[:upper:]]/) {
+    # implicit 'ex' metadata set
+
+    $self->{'doc_mdprefix_fields'}->{'ex'}->{$field}++;
+    $self->{'mdprefix_fields'}->{'ex'}->{$field}++;
+    }
+
+}
+
+
 sub infodb {
     my $self = shift (@_);
@@ -351 +381 @@
     my $first = 1;
     my $url = "";
+
+    $self->{'doc_mdprefix_fields'} = {};
+
     while (defined $section) {
     # update a few statistics
@@ -412 +445 @@
         if (!defined $self->{'dontgdbm'}->{$field}) {
             print $handle "<$field>$value\n";
+
+            if ($section eq "")
+            {
+            $self->infodb_metadata_stats($field);
+            }
         }
         }
     }
 
+    if ($section eq "")
+    {
+        my $doc_mdprefix_fields = $self->{'doc_mdprefix_fields'};
+
+        foreach my $prefix (keys %$doc_mdprefix_fields)
+        {
+        print $handle "<metadataset>$prefix\n";
+
+        foreach my $field (keys %{$doc_mdprefix_fields->{$prefix}})
+        {
+            my $val = $doc_mdprefix_fields->{$prefix}->{$field};
+
+            print $handle "<metadatalist-$prefix>$field\n";
+            print $handle "<metadatafreq-$prefix-$field>$val\n";
+        }
+
+        }
+    }
 
     # If doc_obj reconstructed from GDBM file then no need to 

gsdl/trunk/perllib/lucenebuildproc.pm

@@ -398 +398 @@
 # /** process() **/
 
+
+# Following methods seem to be no different to those defined in basebuildproc.pm
+# From inspection, it looks like these ones can be removed
+
+
 sub get_num_docs {
     my $self = shift (@_);

gsdl/trunk/perllib/mgppbuilder.pm

@@ -634 +634 @@
 
     # do the collection info
-    print $handle "[collection]\n";
-    
+    $self->output_collection_meta_start($handle);
+    $self->output_collection_meta_sets($handle);
+
     # first do the collection meta stuff - everything without a dot
     my $collmetadefined = 0;
@@ -701 +702 @@
     }
     }
-    print $handle $lang_entry;
-    # end the collection entry
-    print $handle "\n" . ('-' x 70) . "\n";        
-
-
+    print $handle "$lang_entry\n";
+
+    $self->output_collection_meta_end($handle);
 }