source: gs2-extensions/parallel-building/trunk/src/bin/script/test_fileutils.pl@ 27512

Last change on this file since 27512 was 27512, checked in by jmt12, 11 years ago

Adding in a special test for measuring the effect of altering ThriftFS buffer size
  • Property svn:executable set to *
File size: 28.0 KB
Line 
1#!/usr/bin/perl
2
3use strict;
4use warnings;
5no warnings 'once';
6
7# Configure
8my $user = 'jmt12';
9my $redirect_errors = 0;
10my $display_errors = 0;
11
12my $test_localfs = 0;
13my $test_hdthriftfs = 1;
14my $test_hdfsshell = 0;
15
16# Globals
17my $base_path;
18my $test_count;
19my $pass_count;
20my $skip_count;
21my $start_time;
22my $total_time;
23my $error_messages;
24
25my $sample_str1 = "Neque porro quisquam est qui dolorem ipsum quia dolor sit amet,\nconsectetur,\nadipisci velit...\n";
26# This was going to be used for append (since Thrift has an append function)
27# but it turns out append is dependant upon HDFS being configured to allow
28# append - and mine isn't. Left here for those whom don't speak latin!
29my $sample_str2 = "[Translation]\nThere is no one who loves pain itself,\nwho seeks after it and wants to have it,\nsimply because it is pain...";
30
31# Requires setup.bash to have been sourced
32BEGIN
33{
34 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
35 die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
36 die "Not supported under windows\n" unless $ENV{'GSDLOS'} !~ /^windows$/i;
37 # Ensure Greenstone Perl locations are in INC
38 unshift (@INC, $ENV{'GSDLHOME'} . '/perllib');
39 unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/cpan');
40 if (defined $ENV{'GSDLEXTS'})
41 {
42 my @extensions = split(/:/, $ENV{'GSDLEXTS'});
43 foreach my $e (@extensions)
44 {
45 my $ext_prefix = $ENV{'GSDLHOME'} . '/ext/' . $e;
46 unshift (@INC, $ext_prefix . '/perllib');
47 unshift (@INC, $ext_prefix . '/perllib/cpan');
48 }
49 }
50
51 # Manually installed CPAN package in GEXT*INSTALL
52 # - parse up version number
53 my ($major, $minor, $revision) = $] =~ /(\d+)\.(\d\d\d)(\d\d\d)/;
54 # - get rid of leading zeros by making them integers
55 $major += 0;
56 $minor += 0;
57 $revision += 0;
58 # - and add to Perl's path
59 unshift (@INC, $ENV{'GEXTPARALLELBUILDING_INSTALLED'} . '/lib/perl/' . $major . '.' . $minor . '.' . $revision);
60}
61
62use Cwd;
63use Devel::Peek;
64
65use FileUtils;
66
67# populate globals
68$base_path = $ENV{'GEXTPARALLELBUILDING'};
69
70# SPECIAL TESTS
71if (defined $ARGV[0])
72{
73 if ($ARGV[0] eq "-buffersize")
74 {
75 print STDERR "Exploring the affect of ThriftBuffer size on transfer speed\n";
76 my $filename = 'WekaManual-3-6-7.pdf';
77 my $local_path = $base_path . '/resources/' . $filename;
78 my $thrift_path = 'HDThriftFS:///user/' . $user. '/' . $filename;
79 my $tmp_path = '/tmp/' . $filename;
80 print STDERR " - copying " . $filename . " (" . &FileUtils::fileSize($local_path) . " bytes)\n";
81 $start_time = time();
82 print STDERR " - copy to Thrift... ";
83 &FileUtils::copyFiles($local_path, $thrift_path);
84 print STDERR "Done in " . (time() - $start_time) . " seconds\n";
85 $start_time = time();
86 print STDERR " - copy from Thrift...";
87 &FileUtils::copyFiles($thrift_path, $tmp_path);
88 print STDERR "Done in " . (time() - $start_time) . " seconds\n";
89 my $result = `diff "$local_path" "$tmp_path"`;
90 $test_count = 0;
91 &testAction(\$test_count, "Comparing binary files (roundtrip)", '', $result);
92 &FileUtils::removeFiles($thrift_path);
93 &FileUtils::removeFiles($tmp_path);
94 exit;
95 }
96}
97
98print "\n";
99print "============================= Test FileUtils ============================\n";
100print "Test the FileUtils module to ensure it correctly manages local and HDFS\n";
101print "based files. HDFS support is provided by two different drivers: HDFSShell\n";
102print "uses repeated calls to the CLI Hadoop program, while HDThriftFS creates a\n";
103print "client socket connection to a running Thrift server.\n";
104print "=========================================================================\n";
105print "\n";
106
107if ($redirect_errors)
108{
109 open SAVEERR, ">&STDERR";
110 close STDERR;
111 open STDERR, ">", \$error_messages or die "What the hell?\n";
112}
113
114if ($test_localfs)
115{
116 $start_time = time();
117 print "A. Local File System\n";
118 $test_count = 0;
119 $pass_count = 0;
120 $skip_count = 0;
121 my $test_dir = '/tmp/fileutils';
122 # From this test on you are safe to assume the testing directory exists
123 &testMakeDirectory($test_dir);
124 &testFilePutContents($test_dir); # creates alpha.txt
125 &testFileGetContents($test_dir); # removes alpha.txt
126 &testFileExists($test_dir);
127 &testDirectoryExists($test_dir);
128 &testFilePermissions($test_dir);
129 &testFilenameConcatenate('/home/gsdl/collect/test/etc/collect.cfg', # target path
130 '/', 'home','gsdl','collect','test','etc','collect.cfg', #parts
131 );
132 &testOpenFileHandle($test_dir);
133 &testFileSize($test_dir);
134 &testModificationTime($test_dir);
135 &testDifferentFiles($test_dir);
136 &testReadDirectory($test_dir);
137 &testTransferFiles($test_dir, 'move');
138 &testTransferFiles($test_dir, 'copy');
139 &testLinkFile($test_dir, $base_path, 'soft');
140 &testLinkFile($test_dir, $base_path, 'hard');
141 $skip_count += &skipAction(\$test_count, "Binary file transfer test (roundtrip)");
142 &testRemoveFiles($test_dir);
143 &testRemoveFilesFiltered($test_dir);
144 &testRemoveFilesRecursive($test_dir);
145 print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
146 $total_time = time() - $start_time;
147 print "Time: " . $total_time . " seconds\n\n";
148 # Full cleanup before we start the next round of tests
149 &FileUtils::removeFilesRecursive($test_dir);
150}
151
152if ($test_hdthriftfs)
153{
154 $start_time = time();
155 print "B. HDFS using Thrift\n";
156 my $thrift_fileutils_path = 'HDThriftFS:///user/' . $user. '/fileutils';
157 $test_count = 0;
158 $pass_count = 0;
159 $skip_count = 0;
160 &testFilenameConcatenate('HDThriftFS:///home/gsdl/collect/test/etc/collect.cfg', # target path
161 'HDThriftFS://', 'home','gsdl','collect','test','etc','collect.cfg', #parts
162 );
163 &testMakeDirectory($thrift_fileutils_path);
164 &testFilePutContents($thrift_fileutils_path); # Leaves alpha.txt
165 &testFileGetContents($thrift_fileutils_path); # Removes alpha.txt
166 &testFileExists($thrift_fileutils_path);
167 &testDirectoryExists($thrift_fileutils_path);
168 &testFilePermissions($thrift_fileutils_path);
169 &testOpenFileHandle($thrift_fileutils_path);
170 &testFileSize($thrift_fileutils_path);
171 &testModificationTime($thrift_fileutils_path);
172 &testDifferentFiles($thrift_fileutils_path);
173 &testReadDirectory($thrift_fileutils_path);
174 &testTransferFiles($thrift_fileutils_path, 'move', '/tmp');
175 &testTransferFiles($thrift_fileutils_path, 'copy', '/tmp');
176 &testLinkFile($thrift_fileutils_path, '', 'soft');
177 &testLinkFile($thrift_fileutils_path, '', 'hard');
178 &testBinaryTransfer($thrift_fileutils_path, $base_path);
179 &testRemoveFiles($thrift_fileutils_path);
180 &testRemoveFilesFiltered($thrift_fileutils_path);
181 &testRemoveFilesRecursive($thrift_fileutils_path);
182 print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
183 $total_time = time() - $start_time;
184 print "Time: " . $total_time . " seconds\n\n";
185 # Cleanup
186 &FileUtils::removeFilesRecursive($thrift_fileutils_path);
187}
188
189if ($test_hdfsshell)
190{
191 $start_time = time();
192 print "C. HDFS using Shell\n";
193 my $hdfs_fileutils_path = 'HDFSShell:///user/' . $user. '/fileutils';
194 $test_count = 0;
195 $pass_count = 0;
196 $skip_count = 0;
197 &testFilenameConcatenate('HDFSShell:///home/gsdl/collect/test/etc/collect.cfg', # target path
198 'HDFSShell://', 'home','gsdl','collect','test','etc','collect.cfg', #parts
199 );
200 &testMakeDirectory($hdfs_fileutils_path);
201 &testFilePutContents($hdfs_fileutils_path); # Leaves alpha.txt
202 &testFileGetContents($hdfs_fileutils_path); # Removes alpha.txt
203 &testFileExists($hdfs_fileutils_path);
204 &testDirectoryExists($hdfs_fileutils_path);
205 &testFilePermissions($hdfs_fileutils_path);
206 &testOpenFileHandle($hdfs_fileutils_path);
207 &testFileSize($hdfs_fileutils_path);
208 &testModificationTime($hdfs_fileutils_path);
209 &testDifferentFiles($hdfs_fileutils_path);
210 &testReadDirectory($hdfs_fileutils_path);
211 &testTransferFiles($hdfs_fileutils_path, 'move', '/tmp');
212 &testTransferFiles($hdfs_fileutils_path, 'copy', '/tmp');
213 &testLinkFile($hdfs_fileutils_path, '', 'soft');
214 &testLinkFile($hdfs_fileutils_path, '', 'hard');
215 &testBinaryTransfer($hdfs_fileutils_path, $base_path);
216 &testRemoveFiles($hdfs_fileutils_path);
217 &testRemoveFilesFiltered($hdfs_fileutils_path);
218 &testRemoveFilesRecursive($hdfs_fileutils_path);
219 print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
220 $total_time = time() - $start_time;
221 print "Time: " . $total_time . " seconds\n\n";
222 # Cleanup
223 &FileUtils::removeFilesRecursive($hdfs_fileutils_path);
224}
225
226if ($redirect_errors)
227{
228 close STDERR;
229 open STDERR, ">&SAVEERR";
230 if ($display_errors)
231 {
232 print "Error Messages\n";
233 print $error_messages;
234 print "\n\n";
235 }
236}
237
238exit;
239
240# /** A test that doesn't actually test anything, but always skips. Used as a
241# * placeholder where certain tests aren't applicable.
242# */
243sub skipAction
244{
245 my ($test_count_ref, $description) = @_;
246 $$test_count_ref++;
247 printf("% 2d. %s: Skipped\n", $$test_count_ref, $description);
248 return 1;
249}
250
251sub subtestAction
252{
253 my ($expected_result, $actual_result, $debug) = @_;
254 if (defined $debug && $debug)
255 {
256 print "subtestAction(expected:$expected_result, actual:$actual_result)\n";
257 }
258 my $result = 'Fail';
259 if (defined $actual_result && (('' . $expected_result) eq ('' . $actual_result)))
260 {
261 $result = 'Pass';
262 }
263 return ('Pass' eq $result);
264}
265
266sub testAction
267{
268 my ($test_count_ref, $description, $expected_result, $actual_result) = @_;
269 $$test_count_ref++;
270 my $result = 'Fail';
271 if (defined $actual_result && (('' . $expected_result) eq ('' . $actual_result)))
272 {
273 $result = 'Pass';
274 }
275 else
276 {
277 $result .= ' (' . $expected_result . ' != ' . $actual_result . ')';
278 }
279 printf("% 2d. %s: %s\n", $$test_count_ref, $description, $result);
280 return ('Pass' eq $result);
281}
282
283################################################################################
284##### TESTS
285################################################################################
286
287## @function testBinaryTransfer()
288#
289sub testBinaryTransfer
290{
291 my ($dir, $base_dir, $file) = @_;
292 if (!defined $file)
293 {
294 $file = 'aurora_australus.jpg';
295 }
296 # setup
297 my ($extension) = $file =~ /\.([^\.]+)$/;
298 my $local_image_path = $base_path . '/resources/' . $file;
299 my $thrift_copy_path = $dir . '/test_binary_transfer.' . $extension;
300 my $local_copy_path = '/tmp/test_binary_transfer.' . $extension;
301 # actions - roundtrip copy of files
302 &FileUtils::copyFiles($local_image_path, $thrift_copy_path);
303 &FileUtils::copyFiles($thrift_copy_path, $local_copy_path);
304 # test
305 my $result = `diff $local_image_path $local_copy_path`;
306 $pass_count += &testAction(\$test_count, "Binary file transfer test (roundtrip)", '', $result);
307 # cleanup
308 &FileUtils::removeFiles($thrift_copy_path, $local_copy_path);
309}
310## testBinaryTransfer()
311
312
313## @function testDifferentFiles
314#
315sub testDifferentFiles
316{
317 my ($dir) = @_;
318 my $patha = $dir . '/alpha.txt';
319 &FileUtils::filePutContents($patha, 'alpha');
320 my $pathb = $dir . '/beta.txt';
321 &FileUtils::filePutContents($pathb, 'beta');
322 $pass_count += &testAction(\$test_count, "differentFiles() comparing a file to itself", 0, &FileUtils::differentFiles($patha, $patha));
323 $pass_count += &testAction(\$test_count, "differentFiles() comparing a directory to itself", 0, &FileUtils::differentFiles($dir, $dir));
324 $pass_count += &testAction(\$test_count, "differentFiles() comparing different files", 1, &FileUtils::differentFiles($patha, $pathb));
325 $pass_count += &testAction(\$test_count, "differentFiles() comparing a file to a directory", 1, &FileUtils::differentFiles($patha, $dir));
326 # -cleanup
327 &FileUtils::removeFiles($patha, $pathb);
328}
329# /** testDifferentFiles() **/
330
331sub testDirectoryExists
332{
333 my ($patha, $pathb, $pathc, $pathd) = @_;
334 $pass_count += &testAction(\$test_count, "directoryExists() for existing directory", 1, &FileUtils::directoryExists($patha));
335 $pass_count += &testAction(\$test_count, "directoryExists() for existing file (should fail)", 0, &FileUtils::directoryExists($pathb));
336 $pass_count += &testAction(\$test_count, "directoryExists() for directory that doesn't exist (should fail)", 0, &FileUtils::directoryExists($pathc));
337 $pass_count += &testAction(\$test_count, "directoryExists() for file that doesn't exist (should fail)", 0, &FileUtils::directoryExists($pathd));
338}
339
340
341## @function testFileExists()
342#
343sub testFileExists
344{
345 my ($dir) = @_;
346 my $alpha_path = $dir . '/alpha.txt';
347 &FileUtils::filePutContents($alpha_path, 'alpha');
348 $pass_count += &testAction(\$test_count, "fileExists() for existing file", 1, &FileUtils::fileExists($alpha_path));
349 my $beta_path = $dir . '/beta.txt';
350 $pass_count += &testAction(\$test_count, "fileExists() for file that doesn't exist (should fail)", 0, &FileUtils::fileExists($beta_path));
351}
352## testFileExists()
353
354
355## @function testFileGetContents()
356#
357sub testFileGetContents
358{
359 my ($dir) = @_;
360 my $apath = $dir . '/alpha.txt';
361 $pass_count += &testAction(\$test_count, "fileGetContents() to read an entire file as a string", $sample_str1, &FileUtils::fileGetContents($apath));
362 &FileUtils::removeFiles($apath); # just have to hope this works
363}
364## testFileGetContents()
365
366
367## @function testFilePermissions()
368#
369sub testFilePermissions
370{
371 my ($dir) = @_;
372 my $patha = $dir . '/alpha.txt';
373 &FileUtils::filePutContents($patha, 'alpha');
374 $pass_count += &testAction(\$test_count, "canRead() testing read permission on a file I own", 1, &FileUtils::canRead($patha));
375 $pass_count += &testAction(\$test_count, "canRead() testing read permission on a file in my group", 1, &FileUtils::canRead($patha, 'johnsmith', 'supergroup'));
376 $pass_count += &testAction(\$test_count, "canRead() testing global access read permission on a file", 1, &FileUtils::canRead($patha, 'johnsmith', 'othergroup'));
377
378 $skip_count += &skipAction(\$test_count, "canWrite() testing write permission on a file");
379 $skip_count += &skipAction(\$test_count, "canExecute() testing execute permission on a file");
380 # - cleanup
381 &FileUtils::removeFiles($patha);
382}
383## testFilePermissions()
384
385
386## @function testFilePutContents()
387#
388sub testFilePutContents
389{
390 my ($dir) = @_;
391 my $apath = $dir . '/alpha.txt';
392 $pass_count += &testAction(\$test_count, "filePutContents() to write a string direct to a file", 1, &FileUtils::filePutContents($apath, $sample_str1));
393}
394## testFilePutContents()
395
396
397## @function testFileSize()
398#
399sub testFileSize
400{
401 my ($dir) = @_;
402 my $apath = $dir . '/alpha.txt';
403 my $astr = 'alpha';
404 &FileUtils::filePutContents($apath, $astr);
405 $pass_count += &testAction(\$test_count, "fileSize() to determine a file's size in bytes", length($astr), &FileUtils::fileSize($apath));
406 &FileUtils::removeFiles($apath);
407}
408## testFileSize()
409
410
411sub testFilenameConcatenate
412{
413 my $target = shift(@_);
414 $pass_count += &testAction(\$test_count, "filenameConcatenate() to combine file paths", $target, &FileUtils::filenameConcatenate(@_));
415}
416
417sub testLinkFile
418{
419 my ($dir, $base_path, $mode) = @_;
420 if (!&FileUtils::supportsSymbolicLink($dir))
421 {
422 &FileUtils::printWarning('Symbolic links not supported so files will be copied instead');
423 }
424
425 # - init
426 my $sub_pass_count = 0;
427 my $patha = $dir . '/alpha.txt';
428 &FileUtils::filePutContents($patha, 'alpha');
429 my $pathb = $dir . '/beta.txt';
430 # - subtests
431 if ($mode eq 'soft')
432 {
433 $sub_pass_count += &subtestAction(1, &FileUtils::softLink($patha, $pathb));
434 # If the driver doesn't support symbolic links, then the following function
435 # will actually end up triggering a '-e' - which is true even for hardlinks
436 $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
437 }
438 else
439 {
440 $sub_pass_count += &subtestAction(1, &FileUtils::hardLink($patha, $pathb));
441 if (&FileUtils::supportsSymbolicLink($pathb))
442 {
443 $sub_pass_count += &subtestAction(0, &FileUtils::isSymbolicLink($pathb));
444 }
445 # If the driver doesn't support symbolic links, then the following function
446 # will actually end up triggering a '-e' - which is true even for hardlinks
447 else
448 {
449 $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
450 }
451 }
452 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($patha));
453 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($pathb));
454 $sub_pass_count += &subtestAction('alpha', &FileUtils::fileGetContents($pathb));
455 # - test
456 $pass_count += &testAction(\$test_count, $mode . "Link() " . $mode . " symbolic linking a single file using relative path", 5, $sub_pass_count);
457 # - cleanup
458 &FileUtils::removeFiles($pathb);
459
460 # - init
461 if ($base_path ne '')
462 {
463 my $depth = split(/[\/\\]/, $base_path) - 1;
464 my $rel_prefix = '..';
465 for (my $i = 1; $i < $depth; $i++)
466 {
467 $rel_prefix .= '/..';
468 }
469 my $rel_patha = $rel_prefix . $patha;
470 my $rel_pathb = $rel_prefix . $pathb;
471 $sub_pass_count = 0;
472 # - subtests
473 if ($mode eq 'soft')
474 {
475 $sub_pass_count += &subtestAction(1, &FileUtils::softLink($rel_patha, $rel_pathb));
476 $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
477 }
478 else
479 {
480 $sub_pass_count += &subtestAction(1, &FileUtils::hardLink($rel_patha, $rel_pathb));
481 $sub_pass_count += &subtestAction(0, &FileUtils::isSymbolicLink($pathb));
482 }
483 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($patha));
484 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($pathb));
485 $sub_pass_count += &subtestAction('alpha', &FileUtils::fileGetContents($pathb));
486 # - test
487 $pass_count += &testAction(\$test_count, $mode . "Link() " . $mode . " symbolic linking a single file forcing absolute path", 5, $sub_pass_count);
488 # - cleanup
489 &FileUtils::removeFiles($pathb, $patha);
490 }
491 else
492 {
493 $skip_count += &skipAction(\$test_count, $mode . "Link() relative paths not supported in this filesystem");
494 }
495}
496
497sub testMakeDirectory
498{
499 my ($path) = @_;
500 $pass_count += &testAction(\$test_count, 'makeDirectory() to create a new directory', 1, &FileUtils::makeDirectory($path));
501 $pass_count += &testAction(\$test_count, 'makeDirectory() for an existing directory', 1, &FileUtils::makeDirectory($path));
502
503 my $multiple_dirs_path = $path . '/foo/bar/wibble';
504 $pass_count += &testAction(\$test_count, 'makeAllDirectories() to create several nested directories', 1, &FileUtils::makeAllDirectories($multiple_dirs_path));
505 &FileUtils::removeFilesRecursive($path . '/foo');
506}
507
508sub testModificationTime
509{
510 my ($dir) = @_;
511 # setup
512 my $patha = $dir . '/alpha.txt';
513 my $modification_time = time();
514 &FileUtils::filePutContents($patha, 'Hello World');
515 my $file_modification_time = &FileUtils::modificationTime($patha);
516 # test
517 $pass_count += &testAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc', $modification_time, $file_modification_time);
518 # cleanup
519 &FileUtils::removeFiles($patha);
520}
521
522sub testOpenFileHandle
523{
524 my ($dir) = @_;
525 my $path = $dir . '/alpha.txt';
526 my $dest_str;
527 my $file_handle;
528 # - writing
529 $pass_count += &testAction(\$test_count, "openFileHandle() to open a file handle for writing", 1, &FileUtils::openFileHandle($path, 'w', \$file_handle));
530 print $file_handle $sample_str1;
531 $pass_count += &testAction(\$test_count, "closeFileHandle() to close the writable file handle", 1, &FileUtils::closeFileHandle($path, \$file_handle));
532 # - reading
533 $pass_count += &testAction(\$test_count, "openFileHandle() to open a file handle for reading", 1, &FileUtils::openFileHandle($path, 'r', \$file_handle));
534 while (my $line = <$file_handle>)
535 {
536 $dest_str .= $line;
537 }
538 $pass_count += &testAction(\$test_count, "comparing string written to string read", $sample_str1, $dest_str);
539 $pass_count += &testAction(\$test_count, "closeFileHandle() to close the readable file handle", 1, &FileUtils::closeFileHandle($path, \$file_handle));
540 # - cleanup
541 &FileUtils::removeFiles($path);
542}
543
544sub testReadDirectory
545{
546 my ($dir) = @_;
547 my $apath = $dir . '/alpha.txt';
548 &FileUtils::filePutContents($apath, 'alpha');
549 my $bpath = $dir . '/beta.txt';
550 &FileUtils::filePutContents($bpath, 'beta');
551 my $dpath = $dir . '/delta.txt';
552 &FileUtils::filePutContents($dpath, 'delta');
553 my $file_count = 3;
554 # - read the directory, which will only contain the files we just put there
555 my @files = @{&FileUtils::readDirectory($dir)};
556 my $result_count = 0;
557 foreach my $file (@files)
558 {
559 if ($file !~ /^\./)
560 {
561 $result_count++;
562 }
563 }
564 $pass_count += &testAction(\$test_count, "readDirectory() listing files", $file_count, $result_count);
565 &FileUtils::removeFiles($apath, $bpath, $dpath);
566}
567
568## @function testRemoveFiles()
569#
570sub testRemoveFiles
571{
572 my ($dir) = @_;
573
574 my $patha = $dir . '/alpha.txt';
575 &FileUtils::filePutContents($patha, 'alpha');
576 my $pathb = $dir . '/beta.txt';
577 &FileUtils::softLink($patha, $pathb);
578 my $pathd = $dir . '/delta.txt';
579 &FileUtils::filePutContents($pathd, 'delta');
580 my $pathe = $dir . '/epsilon.txt';
581 &FileUtils::filePutContents($pathe, 'epsilon');
582 my $pathg = $dir . '/gamma.txt';
583 &FileUtils::filePutContents($pathg, 'gamma');
584 my $patht = $dir . '/tau.txt';
585 my $dirm = $dir .'/mu';
586 &FileUtils::makeDirectory($dirm);
587 # - tests
588 $pass_count += &testAction(\$test_count, "removeFiles() for existing file", 1, &FileUtils::removeFiles($patha));
589 $pass_count += &testAction(\$test_count, "removeFiles() for soft symbolic link", 1, &FileUtils::removeFiles($pathb));
590 $pass_count += &testAction(\$test_count, "removeFiles() for multiple existing files", 3, &FileUtils::removeFiles($pathd, $pathe, $pathg));
591 $pass_count += &testAction(\$test_count, "removeFiles() for file that doesn't exist (should fail)", 0, &FileUtils::removeFiles($patht));
592 $pass_count += &testAction(\$test_count, "removeFiles() for existing directory (should fail)", 0, &FileUtils::removeFiles($dirm));
593 # - cleanup
594 &FileUtils::removeFilesRecursive($dirm);
595}
596## testRemoveFiles()
597
598
599## @function testRemoveFilesFiltered()
600#
601sub testRemoveFilesFiltered
602{
603 my ($dir) = @_;
604 # - setup
605 my $patha = $dir . '/alpha.txt';
606 &FileUtils::filePutContents($patha, 'alpha');
607 &FileUtils::filePutContents($dir . '/beta.tmp','beta');
608 my $pathd = $dir . '/delta.txt';
609 &FileUtils::filePutContents($pathd, 'delta');
610 &FileUtils::filePutContents($dir . '/gamma.tmp','gamma');
611 &FileUtils::filePutContents($dir . '/epsilon.txt','epsilon');
612 &FileUtils::filePutContents($dir . '/eta.txt','eta');
613 # - tests
614 $pass_count += &testAction(\$test_count, "filteredRemove() with accept regular expression", 2, &FileUtils::removeFilesFiltered($dir, '\.tmp$', undef));
615 $pass_count += &testAction(\$test_count, "filteredRemove() with accept and reject regular expression", 2, &FileUtils::removeFilesFiltered($dir, '\.txt$', '(alpha|delta)'));
616 # - cleanup
617 &FileUtils::removeFiles($patha, $pathd);
618}
619## testRemoveFilesFiltered()
620
621
622## @function testRemoveFilesRecursive()
623#
624sub testRemoveFilesRecursive
625{
626 my ($dir) = @_;
627 # - setup
628 my $dira = $dir . '/alpha';
629 &FileUtils::makeDirectory($dira);
630 &FileUtils::filePutContents($dira . '/alpha.txt', 'alpha');
631 &FileUtils::filePutContents($dira . '/beta.txt', 'beta');
632 my $dirb = $dira . '/beta';
633 &FileUtils::makeDirectory($dirb);
634 &FileUtils::filePutContents($dirb . '/gamma.txt', 'gamma');
635 # - test
636 $pass_count += &testAction(\$test_count, "removeFilesRecursive() for directory containing files", 5, &FileUtils::removeFilesRecursive($dira));
637}
638## testRemoveFilesRecursive()
639
640
641## @function testTransferFiles()
642#
643sub testTransferFiles
644{
645 my ($dir, $mode, $local_dir) = @_;
646 my $verb = 'moving';
647 if ($mode eq 'copy')
648 {
649 $verb = 'copying';
650 }
651 # Move a single local file to a new local location
652 # - setup
653 my $apath = $dir . '/alpha.txt';
654 &FileUtils::filePutContents($apath,'alpha');
655 my $bpath = $dir . '/beta.txt';
656 my $sub_pass_count = 0;
657 # - subtests
658 if ($mode eq 'move')
659 {
660 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($apath, $bpath));
661 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($apath));
662 }
663 else
664 {
665 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($apath, $bpath));
666 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
667 }
668 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($bpath));
669
670 # - test
671 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " a single file", 3, $sub_pass_count);
672 # - cleanup
673 &FileUtils::removeFiles($bpath);
674
675 # Move multiple local files into a single local directory
676 # - setup
677 my $dpath = $dir . '/delta.txt';
678 &FileUtils::filePutContents($dpath, 'delta');
679 my $gpath = $dir . '/gamma.txt';
680 &FileUtils::filePutContents($gpath, 'gamma');
681 my $move_dir = $dir . '/movedir';
682 &FileUtils::makeDirectory($move_dir);
683 $sub_pass_count = 0;
684 # - subtests
685 if ($mode eq 'move')
686 {
687 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($dpath, $gpath, $move_dir));
688 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($dpath));
689 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($gpath));
690 }
691 else
692 {
693 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($dpath, $gpath, $move_dir));
694 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($dpath));
695 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($gpath));
696 }
697 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($move_dir . '/delta.txt'));
698 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($move_dir . '/gamma.txt'));
699 # - test
700 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a directory", 5, $sub_pass_count);
701 # - cleanup
702 if ($mode ne 'move')
703 {
704 &FileUtils::removeFiles($dpath, $gpath);
705 }
706
707 # Moving multiple files to file
708 if ($mode eq 'move')
709 {
710 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a file (should fail)", 0, &FileUtils::moveFiles($move_dir . '/delta.txt', $move_dir . '/gamma.txt', $bpath));
711 }
712 else
713 {
714 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a file (should fail)", 0, &FileUtils::copyFiles($move_dir . '/delta.txt', $move_dir . '/gamma.txt', $bpath));
715 }
716
717 # - cleanup
718 &FileUtils::removeFilesRecursive($move_dir);
719
720 if (defined $local_dir && -d $local_dir)
721 {
722 &FileUtils::filePutContents($apath, 'alpha');
723 my $zpath = $local_dir . '/fileutils-' . time() . '.txt';
724 # - init
725 $sub_pass_count = 0;
726 # - subtests
727 if ($mode eq 'move')
728 {
729 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($apath, $zpath));
730 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($apath));
731 }
732 else
733 {
734 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($apath, $zpath));
735 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
736 }
737 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($zpath));
738 # - test
739 $pass_count += &testAction(\$test_count, $mode . "Files() from non-local file to local one", 3, $sub_pass_count);
740
741 # If we are copying, we'll need to remove alpha, otherwise the round trip
742 # back to HDFS will fail
743 if ($mode eq 'copy')
744 {
745 &FileUtils::removeFiles($apath);
746 }
747
748 # - init
749 $sub_pass_count = 0;
750 # - subtests
751 if ($mode eq 'move')
752 {
753 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($zpath, $apath));
754 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($zpath));
755 }
756 else
757 {
758 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($zpath, $apath));
759 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($zpath));
760 }
761 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
762
763 # - test
764 $pass_count += &testAction(\$test_count, $mode . "Files() from local file to non-local one", 3, $sub_pass_count);
765 # - cleanup
766 if ($mode eq 'move')
767 {
768 &FileUtils::removeFiles($apath);
769 }
770 else
771 {
772 &FileUtils::removeFiles($apath, $zpath);
773 }
774 }
775 else
776 {
777 # Move a single non-local file to a new local location
778 $skip_count += &skipAction(\$test_count, $mode . "Files() from non-local file to local one");
779 # Move a single local file to a new non-local location
780 $skip_count += &skipAction(\$test_count, $mode . "Files() from local file to non-local one");
781 }
782}
783## testTransferFiles()
784
7851;
Note: See TracBrowser for help on using the repository browser.