Changeset 24683

Show
Ignore:
Timestamp:
28.09.2011 13:20:26 (8 years ago)
Author:
jmt12
Message:

Fixes to process (based predominantly on the new/custom complexmeta and block_hash->{manifest} flags) to allow global scan to listen to manifests

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • gs2-extensions/parallel-building/trunk/src/perllib/plugins/DirectoryPlugin.pm

    r24626 r24683  
    142142    my $self = shift (@_); 
    143143    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_; 
     144 
    144145} 
    145146 
     
    372373    my $outhandle = $self->{'outhandle'}; 
    373374    my $verbosity = $self->{'verbosity'}; 
    374  
     375     
    375376    # Calculate the directory name and ensure it is a directory and 
    376377    # that it is not explicitly blocked. 
     
    465466        next; 
    466467    } 
     468 
     469        # If we are processing a manifest file, then we don't have to guess at 
     470        # what metadata source files to process. Only process those that appear 
     471        # in the metadata_files list in the block_hash. [jmt12] 
     472        if ($block_hash->{'manifest'} eq 'true'  
     473            && !-d $raw_full_filename 
     474            && !$block_hash->{'metadata_files'}->{$raw_full_filename}) 
     475        { 
     476          print $outhandle "**** Not in manifest. Ignoring metadata in $subfile\n" if ($verbosity > 3); 
     477          next; 
     478        } 
     479 
    467480     
    468481    # Recursively read each $raw_subfile 
     
    627640    } 
    628641 
     642        # So, here's my problem. I have an empty archives directory, and a  
     643        # manifest file that says to add 1 out of 2 documents in the import 
     644        # directory. The directory scan finds both files, and doc 1 is in 
     645        # new_files, so that is all good. But doc 2 is also processed, and 
     646        # without any blocking or other rule, also gets imported. 
     647        # If we are processing a Manifest then files (not docs) that aren't in 
     648        # either the new files or reindex files are skipped. 
     649        # We'll also test for metadata files here, even though they should 
     650        # have been handled above - just in case we have the weird case where 
     651        # a file is both a metadata source and a file in its own right. [jmt12] 
     652        if ($block_hash->{'manifest'} eq 'true'  
     653            && !-d $raw_full_filename 
     654            && !$block_hash->{'new_files'}->{$raw_full_filename} 
     655            && !$block_hash->{'reindex_files'}->{$raw_full_filename} 
     656            && !$block_hash->{'metadata_files'}->{$raw_full_filename}) 
     657        { 
     658          print $outhandle "**** Not in manifest. Skipping $subfile\n" if ($verbosity > 3); 
     659          next; 
     660        } 
     661 
    629662    if (defined $self->{'inf_timestamp'}) { 
    630663        # Look to see if it's a completely new file 
     
    653686        } 
    654687        } 
    655     } 
     688        } 
    656689 
    657690    # Recursively read each $subfile