source: gs2-extensions/parallel-building/trunk/src/bin/script/test_fileutils.pl@ 30354

Last change on this file since 30354 was 27585, checked in by jmt12, 11 years ago

The perl on Medusa won't let you immediately treat a returned array in a scalar context (or at least not without casting) - so I'll write this as two lines for clarity sake (readability in Perl... surely not)
  • Property svn:executable set to *
File size: 28.1 KB
Line 
1#!/usr/bin/perl
2
3use strict;
4use warnings;
5no warnings 'once';
6
7# Configure
8my $user = 'jmt12';
9my $redirect_errors = 0;
10my $display_errors = 0;
11
12my $test_localfs = 0;
13my $test_hdthriftfs = 1;
14my $test_hdfsshell = 0;
15
16# Globals
17my $base_path;
18my $test_count;
19my $pass_count;
20my $skip_count;
21my $start_time;
22my $total_time;
23my $error_messages;
24
25my $sample_str1 = "Neque porro quisquam est qui dolorem ipsum quia dolor sit amet,\nconsectetur,\nadipisci velit...\n";
26# This was going to be used for append (since Thrift has an append function)
27# but it turns out append is dependant upon HDFS being configured to allow
28# append - and mine isn't. Left here for those whom don't speak latin!
29my $sample_str2 = "[Translation]\nThere is no one who loves pain itself,\nwho seeks after it and wants to have it,\nsimply because it is pain...";
30
31# Requires setup.bash to have been sourced
32BEGIN
33{
34 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
35 die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
36 die "Not supported under windows\n" unless $ENV{'GSDLOS'} !~ /^windows$/i;
37 # Ensure Greenstone Perl locations are in INC
38 unshift (@INC, $ENV{'GSDLHOME'} . '/perllib');
39 unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/cpan');
40 if (defined $ENV{'GSDLEXTS'})
41 {
42 my @extensions = split(/:/, $ENV{'GSDLEXTS'});
43 foreach my $e (@extensions)
44 {
45 my $ext_prefix = $ENV{'GSDLHOME'} . '/ext/' . $e;
46 unshift (@INC, $ext_prefix . '/perllib');
47 unshift (@INC, $ext_prefix . '/perllib/cpan');
48 }
49 }
50
51 # Manually installed CPAN package in GEXT*INSTALL
52 # - parse up version number
53 my ($major, $minor, $revision) = $] =~ /(\d+)\.(\d\d\d)(\d\d\d)/;
54 # - get rid of leading zeros by making them integers
55 $major += 0;
56 $minor += 0;
57 $revision += 0;
58 # - and add to Perl's path
59 unshift (@INC, $ENV{'GEXTPARALLELBUILDING_INSTALLED'} . '/lib/perl/' . $major . '.' . $minor . '.' . $revision);
60}
61
62use Cwd;
63use Devel::Peek;
64
65use FileUtils;
66
67# populate globals
68$base_path = $ENV{'GEXTPARALLELBUILDING'};
69
70# SPECIAL TESTS
71if (defined $ARGV[0])
72{
73 if ($ARGV[0] eq "-buffersize")
74 {
75 print STDERR "Exploring the effect of ThriftBuffer size on transfer speed\n";
76 my $filename = 'aurora_australus.jpg';
77 if (defined $ARGV[1])
78 {
79 $filename = $ARGV[1];
80 }
81 my $local_path = $base_path . '/resources/' . $filename;
82 my $thrift_path = 'HDThriftFS:///user/' . $user. '/' . $filename;
83 my $tmp_path = '/tmp/' . $filename;
84 print STDERR " - copying " . $filename . " (" . &FileUtils::fileSize($local_path) . " bytes)\n";
85 $start_time = time();
86 print STDERR " - copy to Thrift... ";
87 &FileUtils::copyFiles($local_path, $thrift_path);
88 print STDERR "Done in " . (time() - $start_time) . " seconds\n";
89 $start_time = time();
90 print STDERR " - copy from Thrift...";
91 &FileUtils::copyFiles($thrift_path, $tmp_path);
92 print STDERR "Done in " . (time() - $start_time) . " seconds\n";
93 my $result = `diff "$local_path" "$tmp_path"`;
94 $test_count = 0;
95 &testAction(\$test_count, "Comparing binary files (roundtrip)", '', $result);
96 &FileUtils::removeFiles($thrift_path);
97 &FileUtils::removeFiles($tmp_path);
98 exit;
99 }
100}
101
102print "\n";
103print "============================= Test FileUtils ============================\n";
104print "Test the FileUtils module to ensure it correctly manages local and HDFS\n";
105print "based files. HDFS support is provided by two different drivers: HDFSShell\n";
106print "uses repeated calls to the CLI Hadoop program, while HDThriftFS creates a\n";
107print "client socket connection to a running Thrift server.\n";
108print "=========================================================================\n";
109print "\n";
110
111if ($redirect_errors)
112{
113 open SAVEERR, ">&STDERR";
114 close STDERR;
115 open STDERR, ">", \$error_messages or die "What the hell?\n";
116}
117
118if ($test_localfs)
119{
120 $start_time = time();
121 print "A. Local File System\n";
122 $test_count = 0;
123 $pass_count = 0;
124 $skip_count = 0;
125 my $test_dir = '/tmp/fileutils';
126 # From this test on you are safe to assume the testing directory exists
127 &testMakeDirectory($test_dir);
128 &testFilePutContents($test_dir); # creates alpha.txt
129 &testFileGetContents($test_dir); # removes alpha.txt
130 &testFileExists($test_dir);
131 &testDirectoryExists($test_dir);
132 &testFilePermissions($test_dir);
133 &testFilenameConcatenate('/home/gsdl/collect/test/etc/collect.cfg', # target path
134 '/', 'home','gsdl','collect','test','etc','collect.cfg', #parts
135 );
136 &testOpenFileHandle($test_dir);
137 &testFileSize($test_dir);
138 &testModificationTime($test_dir);
139 &testDifferentFiles($test_dir);
140 &testReadDirectory($test_dir);
141 &testTransferFiles($test_dir, 'move');
142 &testTransferFiles($test_dir, 'copy');
143 &testLinkFile($test_dir, $base_path, 'soft');
144 &testLinkFile($test_dir, $base_path, 'hard');
145 $skip_count += &skipAction(\$test_count, "Binary file transfer test (roundtrip)");
146 &testRemoveFiles($test_dir);
147 &testRemoveFilesFiltered($test_dir);
148 &testRemoveFilesRecursive($test_dir);
149 print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
150 $total_time = time() - $start_time;
151 print "Time: " . $total_time . " seconds\n\n";
152 # Full cleanup before we start the next round of tests
153 &FileUtils::removeFilesRecursive($test_dir);
154}
155
156if ($test_hdthriftfs)
157{
158 $start_time = time();
159 print "B. HDFS using Thrift\n";
160 my $thrift_fileutils_path = 'HDThriftFS:///user/' . $user. '/fileutils';
161 $test_count = 0;
162 $pass_count = 0;
163 $skip_count = 0;
164 &testFilenameConcatenate('HDThriftFS:///home/gsdl/collect/test/etc/collect.cfg', # target path
165 'HDThriftFS://', 'home','gsdl','collect','test','etc','collect.cfg', #parts
166 );
167 &testMakeDirectory($thrift_fileutils_path);
168 &testFilePutContents($thrift_fileutils_path); # Leaves alpha.txt
169 &testFileGetContents($thrift_fileutils_path); # Removes alpha.txt
170 &testFileExists($thrift_fileutils_path);
171 &testDirectoryExists($thrift_fileutils_path);
172 &testFilePermissions($thrift_fileutils_path);
173 &testOpenFileHandle($thrift_fileutils_path);
174 &testFileSize($thrift_fileutils_path);
175 &testModificationTime($thrift_fileutils_path);
176 &testDifferentFiles($thrift_fileutils_path);
177 &testReadDirectory($thrift_fileutils_path);
178 &testTransferFiles($thrift_fileutils_path, 'move', '/tmp');
179 &testTransferFiles($thrift_fileutils_path, 'copy', '/tmp');
180 &testLinkFile($thrift_fileutils_path, '', 'soft');
181 &testLinkFile($thrift_fileutils_path, '', 'hard');
182 &testBinaryTransfer($thrift_fileutils_path, $base_path);
183 &testRemoveFiles($thrift_fileutils_path);
184 &testRemoveFilesFiltered($thrift_fileutils_path);
185 &testRemoveFilesRecursive($thrift_fileutils_path);
186 print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
187 $total_time = time() - $start_time;
188 print "Time: " . $total_time . " seconds\n\n";
189 # Cleanup
190 &FileUtils::removeFilesRecursive($thrift_fileutils_path);
191}
192
193if ($test_hdfsshell)
194{
195 $start_time = time();
196 print "C. HDFS using Shell\n";
197 my $hdfs_fileutils_path = 'HDFSShell:///user/' . $user. '/fileutils';
198 $test_count = 0;
199 $pass_count = 0;
200 $skip_count = 0;
201 &testFilenameConcatenate('HDFSShell:///home/gsdl/collect/test/etc/collect.cfg', # target path
202 'HDFSShell://', 'home','gsdl','collect','test','etc','collect.cfg', #parts
203 );
204 &testMakeDirectory($hdfs_fileutils_path);
205 &testFilePutContents($hdfs_fileutils_path); # Leaves alpha.txt
206 &testFileGetContents($hdfs_fileutils_path); # Removes alpha.txt
207 &testFileExists($hdfs_fileutils_path);
208 &testDirectoryExists($hdfs_fileutils_path);
209 &testFilePermissions($hdfs_fileutils_path);
210 &testOpenFileHandle($hdfs_fileutils_path);
211 &testFileSize($hdfs_fileutils_path);
212 &testModificationTime($hdfs_fileutils_path);
213 &testDifferentFiles($hdfs_fileutils_path);
214 &testReadDirectory($hdfs_fileutils_path);
215 &testTransferFiles($hdfs_fileutils_path, 'move', '/tmp');
216 &testTransferFiles($hdfs_fileutils_path, 'copy', '/tmp');
217 &testLinkFile($hdfs_fileutils_path, '', 'soft');
218 &testLinkFile($hdfs_fileutils_path, '', 'hard');
219 &testBinaryTransfer($hdfs_fileutils_path, $base_path);
220 &testRemoveFiles($hdfs_fileutils_path);
221 &testRemoveFilesFiltered($hdfs_fileutils_path);
222 &testRemoveFilesRecursive($hdfs_fileutils_path);
223 print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
224 $total_time = time() - $start_time;
225 print "Time: " . $total_time . " seconds\n\n";
226 # Cleanup
227 &FileUtils::removeFilesRecursive($hdfs_fileutils_path);
228}
229
230if ($redirect_errors)
231{
232 close STDERR;
233 open STDERR, ">&SAVEERR";
234 if ($display_errors)
235 {
236 print "Error Messages\n";
237 print $error_messages;
238 print "\n\n";
239 }
240}
241
242exit;
243
244# /** A test that doesn't actually test anything, but always skips. Used as a
245# * placeholder where certain tests aren't applicable.
246# */
247sub skipAction
248{
249 my ($test_count_ref, $description) = @_;
250 $$test_count_ref++;
251 printf("% 2d. %s: Skipped\n", $$test_count_ref, $description);
252 return 1;
253}
254
255sub subtestAction
256{
257 my ($expected_result, $actual_result, $debug) = @_;
258 if (defined $debug && $debug)
259 {
260 print "subtestAction(expected:$expected_result, actual:$actual_result)\n";
261 }
262 my $result = 'Fail';
263 if (defined $actual_result && (('' . $expected_result) eq ('' . $actual_result)))
264 {
265 $result = 'Pass';
266 }
267 return ('Pass' eq $result);
268}
269
270sub testAction
271{
272 my ($test_count_ref, $description, $expected_result, $actual_result) = @_;
273 $$test_count_ref++;
274 my $result = 'Fail';
275 if (defined $actual_result && (('' . $expected_result) eq ('' . $actual_result)))
276 {
277 $result = 'Pass';
278 }
279 else
280 {
281 $result .= ' (' . $expected_result . ' != ' . $actual_result . ')';
282 }
283 printf("% 2d. %s: %s\n", $$test_count_ref, $description, $result);
284 return ('Pass' eq $result);
285}
286
287################################################################################
288##### TESTS
289################################################################################
290
291## @function testBinaryTransfer()
292#
293sub testBinaryTransfer
294{
295 my ($dir, $base_dir, $file) = @_;
296 if (!defined $file)
297 {
298 $file = 'aurora_australus.jpg';
299 }
300 # setup
301 my ($extension) = $file =~ /\.([^\.]+)$/;
302 my $local_image_path = $base_path . '/resources/' . $file;
303 my $thrift_copy_path = $dir . '/test_binary_transfer.' . $extension;
304 my $local_copy_path = '/tmp/test_binary_transfer.' . $extension;
305 # actions - roundtrip copy of files
306 &FileUtils::copyFiles($local_image_path, $thrift_copy_path);
307 &FileUtils::copyFiles($thrift_copy_path, $local_copy_path);
308 # test
309 my $result = `diff $local_image_path $local_copy_path`;
310 $pass_count += &testAction(\$test_count, "Binary file transfer test (roundtrip)", '', $result);
311 # cleanup
312 &FileUtils::removeFiles($thrift_copy_path, $local_copy_path);
313}
314## testBinaryTransfer()
315
316
317## @function testDifferentFiles
318#
319sub testDifferentFiles
320{
321 my ($dir) = @_;
322 my $patha = $dir . '/alpha.txt';
323 &FileUtils::filePutContents($patha, 'alpha');
324 my $pathb = $dir . '/beta.txt';
325 &FileUtils::filePutContents($pathb, 'beta');
326 $pass_count += &testAction(\$test_count, "differentFiles() comparing a file to itself", 0, &FileUtils::differentFiles($patha, $patha));
327 $pass_count += &testAction(\$test_count, "differentFiles() comparing a directory to itself", 0, &FileUtils::differentFiles($dir, $dir));
328 $pass_count += &testAction(\$test_count, "differentFiles() comparing different files", 1, &FileUtils::differentFiles($patha, $pathb));
329 $pass_count += &testAction(\$test_count, "differentFiles() comparing a file to a directory", 1, &FileUtils::differentFiles($patha, $dir));
330 # -cleanup
331 &FileUtils::removeFiles($patha, $pathb);
332}
333# /** testDifferentFiles() **/
334
335sub testDirectoryExists
336{
337 my ($patha, $pathb, $pathc, $pathd) = @_;
338 $pass_count += &testAction(\$test_count, "directoryExists() for existing directory", 1, &FileUtils::directoryExists($patha));
339 $pass_count += &testAction(\$test_count, "directoryExists() for existing file (should fail)", 0, &FileUtils::directoryExists($pathb));
340 $pass_count += &testAction(\$test_count, "directoryExists() for directory that doesn't exist (should fail)", 0, &FileUtils::directoryExists($pathc));
341 $pass_count += &testAction(\$test_count, "directoryExists() for file that doesn't exist (should fail)", 0, &FileUtils::directoryExists($pathd));
342}
343
344
345## @function testFileExists()
346#
347sub testFileExists
348{
349 my ($dir) = @_;
350 my $alpha_path = $dir . '/alpha.txt';
351 &FileUtils::filePutContents($alpha_path, 'alpha');
352 $pass_count += &testAction(\$test_count, "fileExists() for existing file", 1, &FileUtils::fileExists($alpha_path));
353 my $beta_path = $dir . '/beta.txt';
354 $pass_count += &testAction(\$test_count, "fileExists() for file that doesn't exist (should fail)", 0, &FileUtils::fileExists($beta_path));
355}
356## testFileExists()
357
358
359## @function testFileGetContents()
360#
361sub testFileGetContents
362{
363 my ($dir) = @_;
364 my $apath = $dir . '/alpha.txt';
365 $pass_count += &testAction(\$test_count, "fileGetContents() to read an entire file as a string", $sample_str1, &FileUtils::fileGetContents($apath));
366 &FileUtils::removeFiles($apath); # just have to hope this works
367}
368## testFileGetContents()
369
370
371## @function testFilePermissions()
372#
373sub testFilePermissions
374{
375 my ($dir) = @_;
376 my $patha = $dir . '/alpha.txt';
377 &FileUtils::filePutContents($patha, 'alpha');
378 $pass_count += &testAction(\$test_count, "canRead() testing read permission on a file I own", 1, &FileUtils::canRead($patha));
379 $pass_count += &testAction(\$test_count, "canRead() testing read permission on a file in my group", 1, &FileUtils::canRead($patha, 'johnsmith', 'supergroup'));
380 $pass_count += &testAction(\$test_count, "canRead() testing global access read permission on a file", 1, &FileUtils::canRead($patha, 'johnsmith', 'othergroup'));
381
382 $skip_count += &skipAction(\$test_count, "canWrite() testing write permission on a file");
383 $skip_count += &skipAction(\$test_count, "canExecute() testing execute permission on a file");
384 # - cleanup
385 &FileUtils::removeFiles($patha);
386}
387## testFilePermissions()
388
389
390## @function testFilePutContents()
391#
392sub testFilePutContents
393{
394 my ($dir) = @_;
395 my $apath = $dir . '/alpha.txt';
396 $pass_count += &testAction(\$test_count, "filePutContents() to write a string direct to a file", 1, &FileUtils::filePutContents($apath, $sample_str1));
397}
398## testFilePutContents()
399
400
401## @function testFileSize()
402#
403sub testFileSize
404{
405 my ($dir) = @_;
406 my $apath = $dir . '/alpha.txt';
407 my $astr = 'alpha';
408 &FileUtils::filePutContents($apath, $astr);
409 $pass_count += &testAction(\$test_count, "fileSize() to determine a file's size in bytes", length($astr), &FileUtils::fileSize($apath));
410 &FileUtils::removeFiles($apath);
411}
412## testFileSize()
413
414
415sub testFilenameConcatenate
416{
417 my $target = shift(@_);
418 $pass_count += &testAction(\$test_count, "filenameConcatenate() to combine file paths", $target, &FileUtils::filenameConcatenate(@_));
419}
420
421sub testLinkFile
422{
423 my ($dir, $base_path, $mode) = @_;
424 if (!&FileUtils::supportsSymbolicLink($dir))
425 {
426 &FileUtils::printWarning('Symbolic links not supported so files will be copied instead');
427 }
428
429 # - init
430 my $sub_pass_count = 0;
431 my $patha = $dir . '/alpha.txt';
432 &FileUtils::filePutContents($patha, 'alpha');
433 my $pathb = $dir . '/beta.txt';
434 # - subtests
435 if ($mode eq 'soft')
436 {
437 $sub_pass_count += &subtestAction(1, &FileUtils::softLink($patha, $pathb));
438 # If the driver doesn't support symbolic links, then the following function
439 # will actually end up triggering a '-e' - which is true even for hardlinks
440 $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
441 }
442 else
443 {
444 $sub_pass_count += &subtestAction(1, &FileUtils::hardLink($patha, $pathb));
445 if (&FileUtils::supportsSymbolicLink($pathb))
446 {
447 $sub_pass_count += &subtestAction(0, &FileUtils::isSymbolicLink($pathb));
448 }
449 # If the driver doesn't support symbolic links, then the following function
450 # will actually end up triggering a '-e' - which is true even for hardlinks
451 else
452 {
453 $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
454 }
455 }
456 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($patha));
457 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($pathb));
458 $sub_pass_count += &subtestAction('alpha', &FileUtils::fileGetContents($pathb));
459 # - test
460 $pass_count += &testAction(\$test_count, $mode . "Link() " . $mode . " symbolic linking a single file using relative path", 5, $sub_pass_count);
461 # - cleanup
462 &FileUtils::removeFiles($pathb);
463
464 # - init
465 if ($base_path ne '')
466 {
467 my @parts = split(/[\/\\]/, $base_path);
468 my $depth = scalar(@parts) - 1;
469 my $rel_prefix = '..';
470 for (my $i = 1; $i < $depth; $i++)
471 {
472 $rel_prefix .= '/..';
473 }
474 my $rel_patha = $rel_prefix . $patha;
475 my $rel_pathb = $rel_prefix . $pathb;
476 $sub_pass_count = 0;
477 # - subtests
478 if ($mode eq 'soft')
479 {
480 $sub_pass_count += &subtestAction(1, &FileUtils::softLink($rel_patha, $rel_pathb));
481 $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
482 }
483 else
484 {
485 $sub_pass_count += &subtestAction(1, &FileUtils::hardLink($rel_patha, $rel_pathb));
486 $sub_pass_count += &subtestAction(0, &FileUtils::isSymbolicLink($pathb));
487 }
488 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($patha));
489 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($pathb));
490 $sub_pass_count += &subtestAction('alpha', &FileUtils::fileGetContents($pathb));
491 # - test
492 $pass_count += &testAction(\$test_count, $mode . "Link() " . $mode . " symbolic linking a single file forcing absolute path", 5, $sub_pass_count);
493 # - cleanup
494 &FileUtils::removeFiles($pathb, $patha);
495 }
496 else
497 {
498 $skip_count += &skipAction(\$test_count, $mode . "Link() relative paths not supported in this filesystem");
499 }
500}
501
502sub testMakeDirectory
503{
504 my ($path) = @_;
505 $pass_count += &testAction(\$test_count, 'makeDirectory() to create a new directory', 1, &FileUtils::makeDirectory($path));
506 $pass_count += &testAction(\$test_count, 'makeDirectory() for an existing directory', 1, &FileUtils::makeDirectory($path));
507
508 my $multiple_dirs_path = $path . '/foo/bar/wibble';
509 $pass_count += &testAction(\$test_count, 'makeAllDirectories() to create several nested directories', 1, &FileUtils::makeAllDirectories($multiple_dirs_path));
510 &FileUtils::removeFilesRecursive($path . '/foo');
511}
512
513sub testModificationTime
514{
515 my ($dir) = @_;
516 # setup
517 my $patha = $dir . '/alpha.txt';
518 my $modification_time = time();
519 &FileUtils::filePutContents($patha, 'Hello World');
520 my $file_modification_time = &FileUtils::modificationTime($patha);
521 # test
522 $pass_count += &testAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc', $modification_time, $file_modification_time);
523 # cleanup
524 &FileUtils::removeFiles($patha);
525}
526
527sub testOpenFileHandle
528{
529 my ($dir) = @_;
530 my $path = $dir . '/alpha.txt';
531 my $dest_str;
532 my $file_handle;
533 # - writing
534 $pass_count += &testAction(\$test_count, "openFileHandle() to open a file handle for writing", 1, &FileUtils::openFileHandle($path, 'w', \$file_handle));
535 print $file_handle $sample_str1;
536 $pass_count += &testAction(\$test_count, "closeFileHandle() to close the writable file handle", 1, &FileUtils::closeFileHandle($path, \$file_handle));
537 # - reading
538 $pass_count += &testAction(\$test_count, "openFileHandle() to open a file handle for reading", 1, &FileUtils::openFileHandle($path, 'r', \$file_handle));
539 while (my $line = <$file_handle>)
540 {
541 $dest_str .= $line;
542 }
543 $pass_count += &testAction(\$test_count, "comparing string written to string read", $sample_str1, $dest_str);
544 $pass_count += &testAction(\$test_count, "closeFileHandle() to close the readable file handle", 1, &FileUtils::closeFileHandle($path, \$file_handle));
545 # - cleanup
546 &FileUtils::removeFiles($path);
547}
548
549sub testReadDirectory
550{
551 my ($dir) = @_;
552 my $apath = $dir . '/alpha.txt';
553 &FileUtils::filePutContents($apath, 'alpha');
554 my $bpath = $dir . '/beta.txt';
555 &FileUtils::filePutContents($bpath, 'beta');
556 my $dpath = $dir . '/delta.txt';
557 &FileUtils::filePutContents($dpath, 'delta');
558 my $file_count = 3;
559 # - read the directory, which will only contain the files we just put there
560 my @files = @{&FileUtils::readDirectory($dir)};
561 my $result_count = 0;
562 foreach my $file (@files)
563 {
564 if ($file !~ /^\./)
565 {
566 $result_count++;
567 }
568 }
569 $pass_count += &testAction(\$test_count, "readDirectory() listing files", $file_count, $result_count);
570 &FileUtils::removeFiles($apath, $bpath, $dpath);
571}
572
573## @function testRemoveFiles()
574#
575sub testRemoveFiles
576{
577 my ($dir) = @_;
578
579 my $patha = $dir . '/alpha.txt';
580 &FileUtils::filePutContents($patha, 'alpha');
581 my $pathb = $dir . '/beta.txt';
582 &FileUtils::softLink($patha, $pathb);
583 my $pathd = $dir . '/delta.txt';
584 &FileUtils::filePutContents($pathd, 'delta');
585 my $pathe = $dir . '/epsilon.txt';
586 &FileUtils::filePutContents($pathe, 'epsilon');
587 my $pathg = $dir . '/gamma.txt';
588 &FileUtils::filePutContents($pathg, 'gamma');
589 my $patht = $dir . '/tau.txt';
590 my $dirm = $dir .'/mu';
591 &FileUtils::makeDirectory($dirm);
592 # - tests
593 $pass_count += &testAction(\$test_count, "removeFiles() for existing file", 1, &FileUtils::removeFiles($patha));
594 $pass_count += &testAction(\$test_count, "removeFiles() for soft symbolic link", 1, &FileUtils::removeFiles($pathb));
595 $pass_count += &testAction(\$test_count, "removeFiles() for multiple existing files", 3, &FileUtils::removeFiles($pathd, $pathe, $pathg));
596 $pass_count += &testAction(\$test_count, "removeFiles() for file that doesn't exist (should fail)", 0, &FileUtils::removeFiles($patht));
597 $pass_count += &testAction(\$test_count, "removeFiles() for existing directory (should fail)", 0, &FileUtils::removeFiles($dirm));
598 # - cleanup
599 &FileUtils::removeFilesRecursive($dirm);
600}
601## testRemoveFiles()
602
603
604## @function testRemoveFilesFiltered()
605#
606sub testRemoveFilesFiltered
607{
608 my ($dir) = @_;
609 # - setup
610 my $patha = $dir . '/alpha.txt';
611 &FileUtils::filePutContents($patha, 'alpha');
612 &FileUtils::filePutContents($dir . '/beta.tmp','beta');
613 my $pathd = $dir . '/delta.txt';
614 &FileUtils::filePutContents($pathd, 'delta');
615 &FileUtils::filePutContents($dir . '/gamma.tmp','gamma');
616 &FileUtils::filePutContents($dir . '/epsilon.txt','epsilon');
617 &FileUtils::filePutContents($dir . '/eta.txt','eta');
618 # - tests
619 $pass_count += &testAction(\$test_count, "filteredRemove() with accept regular expression", 2, &FileUtils::removeFilesFiltered($dir, '\.tmp$', undef));
620 $pass_count += &testAction(\$test_count, "filteredRemove() with accept and reject regular expression", 2, &FileUtils::removeFilesFiltered($dir, '\.txt$', '(alpha|delta)'));
621 # - cleanup
622 &FileUtils::removeFiles($patha, $pathd);
623}
624## testRemoveFilesFiltered()
625
626
627## @function testRemoveFilesRecursive()
628#
629sub testRemoveFilesRecursive
630{
631 my ($dir) = @_;
632 # - setup
633 my $dira = $dir . '/alpha';
634 &FileUtils::makeDirectory($dira);
635 &FileUtils::filePutContents($dira . '/alpha.txt', 'alpha');
636 &FileUtils::filePutContents($dira . '/beta.txt', 'beta');
637 my $dirb = $dira . '/beta';
638 &FileUtils::makeDirectory($dirb);
639 &FileUtils::filePutContents($dirb . '/gamma.txt', 'gamma');
640 # - test
641 $pass_count += &testAction(\$test_count, "removeFilesRecursive() for directory containing files", 5, &FileUtils::removeFilesRecursive($dira));
642}
643## testRemoveFilesRecursive()
644
645
646## @function testTransferFiles()
647#
648sub testTransferFiles
649{
650 my ($dir, $mode, $local_dir) = @_;
651 my $verb = 'moving';
652 if ($mode eq 'copy')
653 {
654 $verb = 'copying';
655 }
656 # Move a single local file to a new local location
657 # - setup
658 my $apath = $dir . '/alpha.txt';
659 &FileUtils::filePutContents($apath,'alpha');
660 my $bpath = $dir . '/beta.txt';
661 my $sub_pass_count = 0;
662 # - subtests
663 if ($mode eq 'move')
664 {
665 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($apath, $bpath));
666 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($apath));
667 }
668 else
669 {
670 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($apath, $bpath));
671 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
672 }
673 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($bpath));
674
675 # - test
676 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " a single file", 3, $sub_pass_count);
677 # - cleanup
678 &FileUtils::removeFiles($bpath);
679
680 # Move multiple local files into a single local directory
681 # - setup
682 my $dpath = $dir . '/delta.txt';
683 &FileUtils::filePutContents($dpath, 'delta');
684 my $gpath = $dir . '/gamma.txt';
685 &FileUtils::filePutContents($gpath, 'gamma');
686 my $move_dir = $dir . '/movedir';
687 &FileUtils::makeDirectory($move_dir);
688 $sub_pass_count = 0;
689 # - subtests
690 if ($mode eq 'move')
691 {
692 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($dpath, $gpath, $move_dir));
693 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($dpath));
694 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($gpath));
695 }
696 else
697 {
698 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($dpath, $gpath, $move_dir));
699 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($dpath));
700 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($gpath));
701 }
702 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($move_dir . '/delta.txt'));
703 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($move_dir . '/gamma.txt'));
704 # - test
705 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a directory", 5, $sub_pass_count);
706 # - cleanup
707 if ($mode ne 'move')
708 {
709 &FileUtils::removeFiles($dpath, $gpath);
710 }
711
712 # Moving multiple files to file
713 if ($mode eq 'move')
714 {
715 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a file (should fail)", 0, &FileUtils::moveFiles($move_dir . '/delta.txt', $move_dir . '/gamma.txt', $bpath));
716 }
717 else
718 {
719 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a file (should fail)", 0, &FileUtils::copyFiles($move_dir . '/delta.txt', $move_dir . '/gamma.txt', $bpath));
720 }
721
722 # - cleanup
723 &FileUtils::removeFilesRecursive($move_dir);
724
725 if (defined $local_dir && -d $local_dir)
726 {
727 &FileUtils::filePutContents($apath, 'alpha');
728 my $zpath = $local_dir . '/fileutils-' . time() . '.txt';
729 # - init
730 $sub_pass_count = 0;
731 # - subtests
732 if ($mode eq 'move')
733 {
734 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($apath, $zpath));
735 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($apath));
736 }
737 else
738 {
739 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($apath, $zpath));
740 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
741 }
742 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($zpath));
743 # - test
744 $pass_count += &testAction(\$test_count, $mode . "Files() from non-local file to local one", 3, $sub_pass_count);
745
746 # If we are copying, we'll need to remove alpha, otherwise the round trip
747 # back to HDFS will fail
748 if ($mode eq 'copy')
749 {
750 &FileUtils::removeFiles($apath);
751 }
752
753 # - init
754 $sub_pass_count = 0;
755 # - subtests
756 if ($mode eq 'move')
757 {
758 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($zpath, $apath));
759 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($zpath));
760 }
761 else
762 {
763 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($zpath, $apath));
764 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($zpath));
765 }
766 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
767
768 # - test
769 $pass_count += &testAction(\$test_count, $mode . "Files() from local file to non-local one", 3, $sub_pass_count);
770 # - cleanup
771 if ($mode eq 'move')
772 {
773 &FileUtils::removeFiles($apath);
774 }
775 else
776 {
777 &FileUtils::removeFiles($apath, $zpath);
778 }
779 }
780 else
781 {
782 # Move a single non-local file to a new local location
783 $skip_count += &skipAction(\$test_count, $mode . "Files() from non-local file to local one");
784 # Move a single local file to a new non-local location
785 $skip_count += &skipAction(\$test_count, $mode . "Files() from local file to non-local one");
786 }
787}
788## testTransferFiles()
789
7901;
Note: See TracBrowser for help on using the repository browser.