Changeset 27514 for gs2-extensions/parallel-building/trunk/src

Timestamp:

2013-05-30T11:08:57+12:00 (11 years ago)

Author:

jmt12

Message:

Altering code to allow configurable length of read/write buffer when moving from local to remote and vice versa

File:

• gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDThriftFS.pm (modified) (6 diffs)

gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDThriftFS.pm

@@ -76 +76 @@
 my $debug = 0;
 my $debug_encoding = 0;
+#my $buffer_length =    4 * 1024; # 4k blocks
+#my $buffer_length =    8 * 1024; # 8k blocks
+#my $buffer_length =   16 * 1024; # 16K blocks
+#my $buffer_length =   32 * 1024; # 32k blocks
+my $buffer_length =   64 * 1024; # 64k blocks
+#my $buffer_length =  128 * 1024; # 128k blocks
+#my $buffer_length =  256 * 1024; # 256k blocks
+#my $buffer_length =  512 * 1024; # 512k blocks
+#my $buffer_length = 1024 * 1024; # 1M blocks
+#my $buffer_length = 2048 * 1024; # 2M blocks
+#my $buffer_length = 4096 * 1024; # 4M blocks
+#my $buffer_length = 8192 * 1024; # 8M blocks
 
 # Globals
@@ -605 +617 @@
     my $data = undef;
     my $offset = 0;
-    my $length = 4096;
     # Read 4K blocks at a time
-    while ($data = $thrift_client->read($fhin, $offset, $length))
+    while ($data = $thrift_client->read($fhin, $offset, $buffer_length))
     {
       $thrift_client->write($fhout, $data);
-      $offset += $length;
-      if (length ($data) < $length)
+      $offset += $buffer_length;
+      if (length ($data) < $buffer_length)
       {
         last;
@@ -653 +664 @@
   my $decoded = '';
   my $fhout = $thrift_client->create($dst_path);
-  while (read($fhin, $decoded, 4096))
+  while (read($fhin, $decoded, $buffer_length))
   {
     if ($debug_encoding)
@@ -714 +725 @@
   my $encoded = undef;
   my $offset = 0;
-  my $length = 4096; # Read 4K blocks at a time
   open($fhout, '>:raw', $dst) or die("Failed to open file for writing: " . $dst);
   my $fhin = $thrift_client->open($src_path);
-  while ($encoded = $thrift_client->read($fhin, $offset, $length))
+  while ($encoded = $thrift_client->read($fhin, $offset, $buffer_length))
   {
     if ($debug_encoding)
@@ -723 +733 @@
       print STDERR "Reading Data: \n=== START ===\n"; Dump($encoded); print STDERR "\n=== END ===\n\n";
     }
-    #my $decoded = decode_base64($encoded);
     my $decoded = MIME::Base91::decode($encoded);
     if ($debug_encoding)
@@ -730 +739 @@
     }
     print $fhout $decoded;
-    last if (length ($encoded) < $length);
-    $offset += $length;
+    if (length ($decoded) < $buffer_length)
+    {
+      last;
+    }
+    else
+    {
+      $offset += $buffer_length;
+    }
   }
   close($fhout);