Changeset 27514

Show
Ignore:
Timestamp:
30.05.2013 11:08:57 (6 years ago)
Author:
jmt12
Message:

Altering code to allow configurable length of read/write buffer when moving from local to remote and vice versa

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDThriftFS.pm

    r27492 r27514  
    7676my $debug = 0; 
    7777my $debug_encoding = 0; 
     78#my $buffer_length =    4 * 1024; # 4k blocks 
     79#my $buffer_length =    8 * 1024; # 8k blocks 
     80#my $buffer_length =   16 * 1024; # 16K blocks 
     81#my $buffer_length =   32 * 1024; # 32k blocks 
     82my $buffer_length =   64 * 1024; # 64k blocks 
     83#my $buffer_length =  128 * 1024; # 128k blocks 
     84#my $buffer_length =  256 * 1024; # 256k blocks 
     85#my $buffer_length =  512 * 1024; # 512k blocks 
     86#my $buffer_length = 1024 * 1024; # 1M blocks 
     87#my $buffer_length = 2048 * 1024; # 2M blocks 
     88#my $buffer_length = 4096 * 1024; # 4M blocks 
     89#my $buffer_length = 8192 * 1024; # 8M blocks 
    7890 
    7991# Globals 
     
    605617    my $data = undef; 
    606618    my $offset = 0; 
    607     my $length = 4096; 
    608619    # Read 4K blocks at a time 
    609     while ($data = $thrift_client->read($fhin, $offset, $length)) 
     620    while ($data = $thrift_client->read($fhin, $offset, $buffer_length)) 
    610621    { 
    611622      $thrift_client->write($fhout, $data); 
    612       $offset += $length; 
    613       if (length ($data) < $length) 
     623      $offset += $buffer_length; 
     624      if (length ($data) < $buffer_length) 
    614625      { 
    615626        last; 
     
    653664  my $decoded = ''; 
    654665  my $fhout = $thrift_client->create($dst_path); 
    655   while (read($fhin, $decoded, 4096)) 
     666  while (read($fhin, $decoded, $buffer_length)) 
    656667  { 
    657668    if ($debug_encoding) 
     
    714725  my $encoded = undef; 
    715726  my $offset = 0; 
    716   my $length = 4096; # Read 4K blocks at a time 
    717727  open($fhout, '>:raw', $dst) or die("Failed to open file for writing: " . $dst); 
    718728  my $fhin = $thrift_client->open($src_path); 
    719   while ($encoded = $thrift_client->read($fhin, $offset, $length)) 
     729  while ($encoded = $thrift_client->read($fhin, $offset, $buffer_length)) 
    720730  { 
    721731    if ($debug_encoding) 
     
    723733      print STDERR "Reading Data: \n=== START ===\n"; Dump($encoded); print STDERR "\n=== END ===\n\n"; 
    724734    } 
    725     #my $decoded = decode_base64($encoded); 
    726735    my $decoded = MIME::Base91::decode($encoded); 
    727736    if ($debug_encoding) 
     
    730739    } 
    731740    print $fhout $decoded; 
    732     last if (length ($encoded) < $length); 
    733     $offset += $length; 
     741    if (length ($decoded) < $buffer_length) 
     742    { 
     743      last; 
     744    } 
     745    else 
     746    { 
     747      $offset += $buffer_length; 
     748    } 
    734749  } 
    735750  close($fhout);