Changeset 16674 for gsdl/trunk/perllib/textcat.pm

Timestamp:

2008-08-07T14:01:27+12:00 (16 years ago)

Author:

ak19

Message:

Added caching for textcat results on filecontents as well: a second map now stores the mapping from filenames to the cached textcat results on the contents of those files.

File:

• gsdl/trunk/perllib/textcat.pm (modified) (2 diffs)

gsdl/trunk/perllib/textcat.pm

@@ -44 +44 @@
 
 # caching related
-my %cache = (); # map of cached text-strings each to array of char-encodings
+my %filename_cache = (); # map of cached text-strings each to array of char-encodings for the strings themselves
+my %filecontents_cache = (); # map of cached filenames to array of char-encodings for the contents of the files
 my $MAX_CACHE_SIZE = 1000;
 
@@ -132 +133 @@
 }
 
-# Same as above, but caches textcat results for subsequent use.
-# The cache is a map of the string to the corresponding array of results 
-# returned by textcat of the possible filename-encodings for that string.
-# Use this method for short strings (such as filenames) rather than huge text
-# files. The cache will be cleared when the max_cache_size is reached, which
-# is MAX_CACHE_SIZE by default or can be specified as a parameter. The cache
-# can also be cleared by a call to clear_cache.
-sub classify_cached {
-    my ($self, $inputref, $opt_freq, $opt_factor, $opt_top, $max_size_of_cache)=@_;
+# Same as above, but caches textcat results on filenames for subsequent use.
+# The cache is a map of the filename to the corresponding filename_encodings
+# (an array of results returned by textcat of the possible filename-encodings 
+# for the indexing filename string itself). 
+# Need to make sure that the filename is only the tailname: no path and no
+# extension (no digits), in order to make optimum use of cached textcat.
+# Textcat is performed on $filename_ref and the results associated with $filename_ref.
+# The cache will be cleared when the max_cache_size is reached, which is
+# MAX_CACHE_SIZE by default or can be specified as a parameter. The cache
+# can also be cleared by a call to clear_filename_cache.
+sub classify_cached_filename {
+    my ($self, $filename_ref, $opt_freq, $opt_factor, $opt_top, $max_size_of_cache)=@_;
     $self->{'max_cache_size'} = $max_size_of_cache if defined $max_size_of_cache;
     
     # if not already in the cache, work it out and put it there
-    if (!defined $cache{$$inputref}) 
+    if (!defined $filename_cache{$$filename_ref}) 
     {
-    if (scalar (keys %cache) >= $self->{'max_cache_size'}) {
-        $self->clear_cache();
-    }
-##  print STDERR "$$inputref is not yet in the cache\n";
-    $cache{$$inputref} = $self->classify($inputref, $opt_freq, $opt_factor, $opt_top);
-    } else {
-##  print STDERR "$$inputref is already in the cache\n";
-    }
-
-##    print STDERR "Count of elements in cache is now: ".scalar (keys %cache)."\n";
+    if (scalar (keys %filename_cache) >= $self->{'max_cache_size'}) {
+        $self->clear_filename_cache();
+    }
+    $filename_cache{$$filename_ref} = $self->classify($filename_ref, $opt_freq, $opt_factor, $opt_top);
+    } 
 
     # return cached array of encodings for the given string
-    return $cache{$$inputref}; 
-}
-
-# Clears the cache (a map of strings to the textcat results for each string).
-sub clear_cache {
+    return $filename_cache{$$filename_ref}; 
+}
+
+# Same as above, but caches textcat results on filecontents for subsequent use.
+# The cache is a map of the filename to an array of possible filename_encodings
+# for the *contents* of the file returned by textcat.
+# Textcat is performed on $contents_ref and the results associated with $filename.
+# The cache will be cleared when the max_cache_size is reached, which is
+# MAX_CACHE_SIZE by default or can be specified as a parameter. The cache
+# can also be cleared by a call to clear_filecontents_cache.
+sub classify_cached_filecontents {
+    my ($self, $contents_ref, $filename, $opt_freq, $opt_factor, $opt_top, $max_size_of_cache)=@_;
+    $self->{'max_cache_size'} = $max_size_of_cache if defined $max_size_of_cache;
+ 
+    # if not already in the cache, work it out and put it there
+    if (!defined $filecontents_cache{$filename}) 
+    {      
+    if (scalar (keys %filecontents_cache) >= $self->{'max_cache_size'}) {
+        $self->clear_filecontents_cache();
+    }
+    $filecontents_cache{$filename} = $self->classify($contents_ref, $opt_freq, $opt_factor, $opt_top);
+    }
+
+    # return cached array of content encodings for the given filename
+    return $filecontents_cache{$filename};
+}
+
+# Clears the filename cache (a map of strings to the textcat results for each string).
+sub clear_filename_cache {
     my $self = shift (@_);
 
-    %cache = undef; # does this suffice to release memory?
-    %cache = ();
+    %filename_cache = undef; # does this suffice to release memory?
+    %filename_cache = ();
+}
+
+# Clears the filecontents cache (a map of filenames to the textcat results on the contents of each file).
+sub clear_filecontents_cache {
+    my $self = shift (@_);
+
+    %filecontents_cache = undef; # does this suffice to release memory?
+    %filecontents_cache = ();
 }