source: gs2-extensions/parallel-building/trunk/src/bin/script/test_fileutils.pl@ 27495

Last change on this file since 27495 was 27481, checked in by jmt12, 11 years ago

Adding makeAllDirectories() (which I'd only implemented in LocalFS) to FileUtils (which in turn calls the Driver specific makeDirectory() recursively) and added test for this function
  • Property svn:executable set to *
File size: 26.8 KB
Line 
1#!/usr/bin/perl
2
3use strict;
4use warnings;
5no warnings 'once';
6
7# Configure
8my $user = 'jmt12';
9my $redirect_errors = 0;
10my $display_errors = 0;
11
12my $test_localfs = 0;
13my $test_hdthriftfs = 1;
14my $test_hdfsshell = 0;
15
16# Globals
17my $base_path;
18my $test_count;
19my $pass_count;
20my $skip_count;
21my $start_time;
22my $total_time;
23my $error_messages;
24
25my $sample_str1 = "Neque porro quisquam est qui dolorem ipsum quia dolor sit amet,\nconsectetur,\nadipisci velit...\n";
26# This was going to be used for append (since Thrift has an append function)
27# but it turns out append is dependant upon HDFS being configured to allow
28# append - and mine isn't. Left here for those whom don't speak latin!
29my $sample_str2 = "[Translation]\nThere is no one who loves pain itself,\nwho seeks after it and wants to have it,\nsimply because it is pain...";
30
31# Requires setup.bash to have been sourced
32BEGIN
33{
34 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
35 die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
36 die "Not supported under windows\n" unless $ENV{'GSDLOS'} !~ /^windows$/i;
37 # Ensure Greenstone Perl locations are in INC
38 unshift (@INC, $ENV{'GSDLHOME'} . '/perllib');
39 unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/cpan');
40 if (defined $ENV{'GSDLEXTS'})
41 {
42 my @extensions = split(/:/, $ENV{'GSDLEXTS'});
43 foreach my $e (@extensions)
44 {
45 my $ext_prefix = $ENV{'GSDLHOME'} . '/ext/' . $e;
46 unshift (@INC, $ext_prefix . '/perllib');
47 unshift (@INC, $ext_prefix . '/perllib/cpan');
48 }
49 }
50
51 # Manually installed CPAN package in GEXT*INSTALL
52 # - parse up version number
53 my ($major, $minor, $revision) = $] =~ /(\d+)\.(\d\d\d)(\d\d\d)/;
54 # - get rid of leading zeros by making them integers
55 $major += 0;
56 $minor += 0;
57 $revision += 0;
58 # - and add to Perl's path
59 unshift (@INC, $ENV{'GEXTPARALLELBUILDING_INSTALLED'} . '/lib/perl/' . $major . '.' . $minor . '.' . $revision);
60}
61
62use Cwd;
63use Devel::Peek;
64
65use FileUtils;
66
67# populate globals
68$base_path = $ENV{'GEXTPARALLELBUILDING'};
69
70print "\n";
71print "============================= Test FileUtils ============================\n";
72print "Test the FileUtils module to ensure it correctly manages local and HDFS\n";
73print "based files. HDFS support is provided by two different drivers: HDFSShell\n";
74print "uses repeated calls to the CLI Hadoop program, while HDThriftFS creates a\n";
75print "client socket connection to a running Thrift server.\n";
76print "=========================================================================\n";
77print "\n";
78
79if ($redirect_errors)
80{
81 open SAVEERR, ">&STDERR";
82 close STDERR;
83 open STDERR, ">", \$error_messages or die "What the hell?\n";
84}
85
86if ($test_localfs)
87{
88 $start_time = time();
89 print "A. Local File System\n";
90 $test_count = 0;
91 $pass_count = 0;
92 $skip_count = 0;
93 my $test_dir = '/tmp/fileutils';
94 # From this test on you are safe to assume the testing directory exists
95 &testMakeDirectory($test_dir);
96 &testFilePutContents($test_dir); # creates alpha.txt
97 &testFileGetContents($test_dir); # removes alpha.txt
98 &testFileExists($test_dir);
99 &testDirectoryExists($test_dir);
100 &testFilePermissions($test_dir);
101 &testFilenameConcatenate('/home/gsdl/collect/test/etc/collect.cfg', # target path
102 '/', 'home','gsdl','collect','test','etc','collect.cfg', #parts
103 );
104 &testOpenFileHandle($test_dir);
105 &testFileSize($test_dir);
106 &testModificationTime($test_dir);
107 &testDifferentFiles($test_dir);
108 &testReadDirectory($test_dir);
109 &testTransferFiles($test_dir, 'move');
110 &testTransferFiles($test_dir, 'copy');
111 &testLinkFile($test_dir, $base_path, 'soft');
112 &testLinkFile($test_dir, $base_path, 'hard');
113 $skip_count += &skipAction(\$test_count, "Binary file transfer test (roundtrip)");
114 &testRemoveFiles($test_dir);
115 &testRemoveFilesFiltered($test_dir);
116 &testRemoveFilesRecursive($test_dir);
117 print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
118 $total_time = time() - $start_time;
119 print "Time: " . $total_time . " seconds\n\n";
120 # Full cleanup before we start the next round of tests
121 &FileUtils::removeFilesRecursive($test_dir);
122}
123
124if ($test_hdthriftfs)
125{
126 $start_time = time();
127 print "B. HDFS using Thrift\n";
128 my $thrift_fileutils_path = 'HDThriftFS:///user/' . $user. '/fileutils';
129 $test_count = 0;
130 $pass_count = 0;
131 $skip_count = 0;
132 &testFilenameConcatenate('HDThriftFS:///home/gsdl/collect/test/etc/collect.cfg', # target path
133 'HDThriftFS://', 'home','gsdl','collect','test','etc','collect.cfg', #parts
134 );
135 &testMakeDirectory($thrift_fileutils_path);
136 &testFilePutContents($thrift_fileutils_path); # Leaves alpha.txt
137 &testFileGetContents($thrift_fileutils_path); # Removes alpha.txt
138 &testFileExists($thrift_fileutils_path);
139 &testDirectoryExists($thrift_fileutils_path);
140 &testFilePermissions($thrift_fileutils_path);
141 &testOpenFileHandle($thrift_fileutils_path);
142 &testFileSize($thrift_fileutils_path);
143 &testModificationTime($thrift_fileutils_path);
144 &testDifferentFiles($thrift_fileutils_path);
145 &testReadDirectory($thrift_fileutils_path);
146 &testTransferFiles($thrift_fileutils_path, 'move', '/tmp');
147 &testTransferFiles($thrift_fileutils_path, 'copy', '/tmp');
148 &testLinkFile($thrift_fileutils_path, '', 'soft');
149 &testLinkFile($thrift_fileutils_path, '', 'hard');
150 &testBinaryTransfer($thrift_fileutils_path, $base_path);
151 &testRemoveFiles($thrift_fileutils_path);
152 &testRemoveFilesFiltered($thrift_fileutils_path);
153 &testRemoveFilesRecursive($thrift_fileutils_path);
154 print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
155 $total_time = time() - $start_time;
156 print "Time: " . $total_time . " seconds\n\n";
157 # Cleanup
158 &FileUtils::removeFilesRecursive($thrift_fileutils_path);
159}
160
161if ($test_hdfsshell)
162{
163 $start_time = time();
164 print "C. HDFS using Shell\n";
165 my $hdfs_fileutils_path = 'HDFSShell:///user/' . $user. '/fileutils';
166 $test_count = 0;
167 $pass_count = 0;
168 $skip_count = 0;
169 &testFilenameConcatenate('HDFSShell:///home/gsdl/collect/test/etc/collect.cfg', # target path
170 'HDFSShell://', 'home','gsdl','collect','test','etc','collect.cfg', #parts
171 );
172 &testMakeDirectory($hdfs_fileutils_path);
173 &testFilePutContents($hdfs_fileutils_path); # Leaves alpha.txt
174 &testFileGetContents($hdfs_fileutils_path); # Removes alpha.txt
175 &testFileExists($hdfs_fileutils_path);
176 &testDirectoryExists($hdfs_fileutils_path);
177 &testFilePermissions($hdfs_fileutils_path);
178 &testOpenFileHandle($hdfs_fileutils_path);
179 &testFileSize($hdfs_fileutils_path);
180 &testModificationTime($hdfs_fileutils_path);
181 &testDifferentFiles($hdfs_fileutils_path);
182 &testReadDirectory($hdfs_fileutils_path);
183 &testTransferFiles($hdfs_fileutils_path, 'move', '/tmp');
184 &testTransferFiles($hdfs_fileutils_path, 'copy', '/tmp');
185 &testLinkFile($hdfs_fileutils_path, '', 'soft');
186 &testLinkFile($hdfs_fileutils_path, '', 'hard');
187 &testBinaryTransfer($hdfs_fileutils_path, $base_path);
188 &testRemoveFiles($hdfs_fileutils_path);
189 &testRemoveFilesFiltered($hdfs_fileutils_path);
190 &testRemoveFilesRecursive($hdfs_fileutils_path);
191 print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
192 $total_time = time() - $start_time;
193 print "Time: " . $total_time . " seconds\n\n";
194 # Cleanup
195 &FileUtils::removeFilesRecursive($hdfs_fileutils_path);
196}
197
198if ($redirect_errors)
199{
200 close STDERR;
201 open STDERR, ">&SAVEERR";
202 if ($display_errors)
203 {
204 print "Error Messages\n";
205 print $error_messages;
206 print "\n\n";
207 }
208}
209
210exit;
211
212# /** A test that doesn't actually test anything, but always skips. Used as a
213# * placeholder where certain tests aren't applicable.
214# */
215sub skipAction
216{
217 my ($test_count_ref, $description) = @_;
218 $$test_count_ref++;
219 printf("% 2d. %s: Skipped\n", $$test_count_ref, $description);
220 return 1;
221}
222
223sub subtestAction
224{
225 my ($expected_result, $actual_result, $debug) = @_;
226 if (defined $debug && $debug)
227 {
228 print "subtestAction(expected:$expected_result, actual:$actual_result)\n";
229 }
230 my $result = 'Fail';
231 if (defined $actual_result && (('' . $expected_result) eq ('' . $actual_result)))
232 {
233 $result = 'Pass';
234 }
235 return ('Pass' eq $result);
236}
237
238sub testAction
239{
240 my ($test_count_ref, $description, $expected_result, $actual_result) = @_;
241 $$test_count_ref++;
242 my $result = 'Fail';
243 if (defined $actual_result && (('' . $expected_result) eq ('' . $actual_result)))
244 {
245 $result = 'Pass';
246 }
247 else
248 {
249 $result .= ' (' . $expected_result . ' != ' . $actual_result . ')';
250 }
251 printf("% 2d. %s: %s\n", $$test_count_ref, $description, $result);
252 return ('Pass' eq $result);
253}
254
255################################################################################
256##### TESTS
257################################################################################
258
259## @function testBinaryTransfer()
260#
261sub testBinaryTransfer
262{
263 my ($dir, $base_dir) = @_;
264 # setup
265 my $local_image_path = $base_path . '/resources/aurora_australus.jpg';
266 my $thrift_copy_path = $dir . '/test_binary_transfer.jpg';
267 my $local_copy_path = '/tmp/test_binary_transfer.jpg';
268 # actions - roundtrip copy of files
269 &FileUtils::copyFiles($local_image_path, $thrift_copy_path);
270 &FileUtils::copyFiles($thrift_copy_path, $local_copy_path);
271 # test
272 my $result = `diff $local_image_path $local_copy_path`;
273 $pass_count += &testAction(\$test_count, "Binary file transfer test (roundtrip)", '', $result);
274 # cleanup
275 &FileUtils::removeFiles($thrift_copy_path, $local_copy_path);
276}
277## testBinaryTransfer()
278
279
280## @function testDifferentFiles
281#
282sub testDifferentFiles
283{
284 my ($dir) = @_;
285 my $patha = $dir . '/alpha.txt';
286 &FileUtils::filePutContents($patha, 'alpha');
287 my $pathb = $dir . '/beta.txt';
288 &FileUtils::filePutContents($pathb, 'beta');
289 $pass_count += &testAction(\$test_count, "differentFiles() comparing a file to itself", 0, &FileUtils::differentFiles($patha, $patha));
290 $pass_count += &testAction(\$test_count, "differentFiles() comparing a directory to itself", 0, &FileUtils::differentFiles($dir, $dir));
291 $pass_count += &testAction(\$test_count, "differentFiles() comparing different files", 1, &FileUtils::differentFiles($patha, $pathb));
292 $pass_count += &testAction(\$test_count, "differentFiles() comparing a file to a directory", 1, &FileUtils::differentFiles($patha, $dir));
293 # -cleanup
294 &FileUtils::removeFiles($patha, $pathb);
295}
296# /** testDifferentFiles() **/
297
298sub testDirectoryExists
299{
300 my ($patha, $pathb, $pathc, $pathd) = @_;
301 $pass_count += &testAction(\$test_count, "directoryExists() for existing directory", 1, &FileUtils::directoryExists($patha));
302 $pass_count += &testAction(\$test_count, "directoryExists() for existing file (should fail)", 0, &FileUtils::directoryExists($pathb));
303 $pass_count += &testAction(\$test_count, "directoryExists() for directory that doesn't exist (should fail)", 0, &FileUtils::directoryExists($pathc));
304 $pass_count += &testAction(\$test_count, "directoryExists() for file that doesn't exist (should fail)", 0, &FileUtils::directoryExists($pathd));
305}
306
307
308## @function testFileExists()
309#
310sub testFileExists
311{
312 my ($dir) = @_;
313 my $alpha_path = $dir . '/alpha.txt';
314 &FileUtils::filePutContents($alpha_path, 'alpha');
315 $pass_count += &testAction(\$test_count, "fileExists() for existing file", 1, &FileUtils::fileExists($alpha_path));
316 my $beta_path = $dir . '/beta.txt';
317 $pass_count += &testAction(\$test_count, "fileExists() for file that doesn't exist (should fail)", 0, &FileUtils::fileExists($beta_path));
318}
319## testFileExists()
320
321
322## @function testFileGetContents()
323#
324sub testFileGetContents
325{
326 my ($dir) = @_;
327 my $apath = $dir . '/alpha.txt';
328 $pass_count += &testAction(\$test_count, "fileGetContents() to read an entire file as a string", $sample_str1, &FileUtils::fileGetContents($apath));
329 &FileUtils::removeFiles($apath); # just have to hope this works
330}
331## testFileGetContents()
332
333
334## @function testFilePermissions()
335#
336sub testFilePermissions
337{
338 my ($dir) = @_;
339 my $patha = $dir . '/alpha.txt';
340 &FileUtils::filePutContents($patha, 'alpha');
341 $pass_count += &testAction(\$test_count, "canRead() testing read permission on a file I own", 1, &FileUtils::canRead($patha));
342 $pass_count += &testAction(\$test_count, "canRead() testing read permission on a file in my group", 1, &FileUtils::canRead($patha, 'johnsmith', 'supergroup'));
343 $pass_count += &testAction(\$test_count, "canRead() testing global access read permission on a file", 1, &FileUtils::canRead($patha, 'johnsmith', 'othergroup'));
344
345 $skip_count += &skipAction(\$test_count, "canWrite() testing write permission on a file");
346 $skip_count += &skipAction(\$test_count, "canExecute() testing execute permission on a file");
347 # - cleanup
348 &FileUtils::removeFiles($patha);
349}
350## testFilePermissions()
351
352
353## @function testFilePutContents()
354#
355sub testFilePutContents
356{
357 my ($dir) = @_;
358 my $apath = $dir . '/alpha.txt';
359 $pass_count += &testAction(\$test_count, "filePutContents() to write a string direct to a file", 1, &FileUtils::filePutContents($apath, $sample_str1));
360}
361## testFilePutContents()
362
363
364## @function testFileSize()
365#
366sub testFileSize
367{
368 my ($dir) = @_;
369 my $apath = $dir . '/alpha.txt';
370 my $astr = 'alpha';
371 &FileUtils::filePutContents($apath, $astr);
372 $pass_count += &testAction(\$test_count, "fileSize() to determine a file's size in bytes", length($astr), &FileUtils::fileSize($apath));
373 &FileUtils::removeFiles($apath);
374}
375## testFileSize()
376
377
378sub testFilenameConcatenate
379{
380 my $target = shift(@_);
381 $pass_count += &testAction(\$test_count, "filenameConcatenate() to combine file paths", $target, &FileUtils::filenameConcatenate(@_));
382}
383
384sub testLinkFile
385{
386 my ($dir, $base_path, $mode) = @_;
387 if (!&FileUtils::supportsSymbolicLink($dir))
388 {
389 &FileUtils::printWarning('Symbolic links not supported so files will be copied instead');
390 }
391
392 # - init
393 my $sub_pass_count = 0;
394 my $patha = $dir . '/alpha.txt';
395 &FileUtils::filePutContents($patha, 'alpha');
396 my $pathb = $dir . '/beta.txt';
397 # - subtests
398 if ($mode eq 'soft')
399 {
400 $sub_pass_count += &subtestAction(1, &FileUtils::softLink($patha, $pathb));
401 # If the driver doesn't support symbolic links, then the following function
402 # will actually end up triggering a '-e' - which is true even for hardlinks
403 $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
404 }
405 else
406 {
407 $sub_pass_count += &subtestAction(1, &FileUtils::hardLink($patha, $pathb));
408 if (&FileUtils::supportsSymbolicLink($pathb))
409 {
410 $sub_pass_count += &subtestAction(0, &FileUtils::isSymbolicLink($pathb));
411 }
412 # If the driver doesn't support symbolic links, then the following function
413 # will actually end up triggering a '-e' - which is true even for hardlinks
414 else
415 {
416 $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
417 }
418 }
419 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($patha));
420 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($pathb));
421 $sub_pass_count += &subtestAction('alpha', &FileUtils::fileGetContents($pathb));
422 # - test
423 $pass_count += &testAction(\$test_count, $mode . "Link() " . $mode . " symbolic linking a single file using relative path", 5, $sub_pass_count);
424 # - cleanup
425 &FileUtils::removeFiles($pathb);
426
427 # - init
428 if ($base_path ne '')
429 {
430 my $depth = split(/[\/\\]/, $base_path) - 1;
431 my $rel_prefix = '..';
432 for (my $i = 1; $i < $depth; $i++)
433 {
434 $rel_prefix .= '/..';
435 }
436 my $rel_patha = $rel_prefix . $patha;
437 my $rel_pathb = $rel_prefix . $pathb;
438 $sub_pass_count = 0;
439 # - subtests
440 if ($mode eq 'soft')
441 {
442 $sub_pass_count += &subtestAction(1, &FileUtils::softLink($rel_patha, $rel_pathb));
443 $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
444 }
445 else
446 {
447 $sub_pass_count += &subtestAction(1, &FileUtils::hardLink($rel_patha, $rel_pathb));
448 $sub_pass_count += &subtestAction(0, &FileUtils::isSymbolicLink($pathb));
449 }
450 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($patha));
451 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($pathb));
452 $sub_pass_count += &subtestAction('alpha', &FileUtils::fileGetContents($pathb));
453 # - test
454 $pass_count += &testAction(\$test_count, $mode . "Link() " . $mode . " symbolic linking a single file forcing absolute path", 5, $sub_pass_count);
455 # - cleanup
456 &FileUtils::removeFiles($pathb, $patha);
457 }
458 else
459 {
460 $skip_count += &skipAction(\$test_count, $mode . "Link() relative paths not supported in this filesystem");
461 }
462}
463
464sub testMakeDirectory
465{
466 my ($path) = @_;
467 $pass_count += &testAction(\$test_count, 'makeDirectory() to create a new directory', 1, &FileUtils::makeDirectory($path));
468 $pass_count += &testAction(\$test_count, 'makeDirectory() for an existing directory', 1, &FileUtils::makeDirectory($path));
469
470 my $multiple_dirs_path = $path . '/foo/bar/wibble';
471 $pass_count += &testAction(\$test_count, 'makeAllDirectories() to create several nested directories', 1, &FileUtils::makeAllDirectories($multiple_dirs_path));
472 &FileUtils::removeFilesRecursive($path . '/foo');
473}
474
475sub testModificationTime
476{
477 my ($dir) = @_;
478 # setup
479 my $patha = $dir . '/alpha.txt';
480 my $modification_time = time();
481 &FileUtils::filePutContents($patha, 'Hello World');
482 my $file_modification_time = &FileUtils::modificationTime($patha);
483 # test
484 $pass_count += &testAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc', $modification_time, $file_modification_time);
485 # cleanup
486 &FileUtils::removeFiles($patha);
487}
488
489sub testOpenFileHandle
490{
491 my ($dir) = @_;
492 my $path = $dir . '/alpha.txt';
493 my $dest_str;
494 my $file_handle;
495 # - writing
496 $pass_count += &testAction(\$test_count, "openFileHandle() to open a file handle for writing", 1, &FileUtils::openFileHandle($path, 'w', \$file_handle));
497 print $file_handle $sample_str1;
498 $pass_count += &testAction(\$test_count, "closeFileHandle() to close the writable file handle", 1, &FileUtils::closeFileHandle($path, \$file_handle));
499 # - reading
500 $pass_count += &testAction(\$test_count, "openFileHandle() to open a file handle for reading", 1, &FileUtils::openFileHandle($path, 'r', \$file_handle));
501 while (my $line = <$file_handle>)
502 {
503 $dest_str .= $line;
504 }
505 $pass_count += &testAction(\$test_count, "comparing string written to string read", $sample_str1, $dest_str);
506 $pass_count += &testAction(\$test_count, "closeFileHandle() to close the readable file handle", 1, &FileUtils::closeFileHandle($path, \$file_handle));
507 # - cleanup
508 &FileUtils::removeFiles($path);
509}
510
511sub testReadDirectory
512{
513 my ($dir) = @_;
514 my $apath = $dir . '/alpha.txt';
515 &FileUtils::filePutContents($apath, 'alpha');
516 my $bpath = $dir . '/beta.txt';
517 &FileUtils::filePutContents($bpath, 'beta');
518 my $dpath = $dir . '/delta.txt';
519 &FileUtils::filePutContents($dpath, 'delta');
520 my $file_count = 3;
521 # - read the directory, which will only contain the files we just put there
522 my @files = @{&FileUtils::readDirectory($dir)};
523 my $result_count = 0;
524 foreach my $file (@files)
525 {
526 if ($file !~ /^\./)
527 {
528 $result_count++;
529 }
530 }
531 $pass_count += &testAction(\$test_count, "readDirectory() listing files", $file_count, $result_count);
532 &FileUtils::removeFiles($apath, $bpath, $dpath);
533}
534
535## @function testRemoveFiles()
536#
537sub testRemoveFiles
538{
539 my ($dir) = @_;
540
541 my $patha = $dir . '/alpha.txt';
542 &FileUtils::filePutContents($patha, 'alpha');
543 my $pathb = $dir . '/beta.txt';
544 &FileUtils::softLink($patha, $pathb);
545 my $pathd = $dir . '/delta.txt';
546 &FileUtils::filePutContents($pathd, 'delta');
547 my $pathe = $dir . '/epsilon.txt';
548 &FileUtils::filePutContents($pathe, 'epsilon');
549 my $pathg = $dir . '/gamma.txt';
550 &FileUtils::filePutContents($pathg, 'gamma');
551 my $patht = $dir . '/tau.txt';
552 my $dirm = $dir .'/mu';
553 &FileUtils::makeDirectory($dirm);
554 # - tests
555 $pass_count += &testAction(\$test_count, "removeFiles() for existing file", 1, &FileUtils::removeFiles($patha));
556 $pass_count += &testAction(\$test_count, "removeFiles() for soft symbolic link", 1, &FileUtils::removeFiles($pathb));
557 $pass_count += &testAction(\$test_count, "removeFiles() for multiple existing files", 3, &FileUtils::removeFiles($pathd, $pathe, $pathg));
558 $pass_count += &testAction(\$test_count, "removeFiles() for file that doesn't exist (should fail)", 0, &FileUtils::removeFiles($patht));
559 $pass_count += &testAction(\$test_count, "removeFiles() for existing directory (should fail)", 0, &FileUtils::removeFiles($dirm));
560 # - cleanup
561 &FileUtils::removeFilesRecursive($dirm);
562}
563## testRemoveFiles()
564
565
566## @function testRemoveFilesFiltered()
567#
568sub testRemoveFilesFiltered
569{
570 my ($dir) = @_;
571 # - setup
572 my $patha = $dir . '/alpha.txt';
573 &FileUtils::filePutContents($patha, 'alpha');
574 &FileUtils::filePutContents($dir . '/beta.tmp','beta');
575 my $pathd = $dir . '/delta.txt';
576 &FileUtils::filePutContents($pathd, 'delta');
577 &FileUtils::filePutContents($dir . '/gamma.tmp','gamma');
578 &FileUtils::filePutContents($dir . '/epsilon.txt','epsilon');
579 &FileUtils::filePutContents($dir . '/eta.txt','eta');
580 # - tests
581 $pass_count += &testAction(\$test_count, "filteredRemove() with accept regular expression", 2, &FileUtils::removeFilesFiltered($dir, '\.tmp$', undef));
582 $pass_count += &testAction(\$test_count, "filteredRemove() with accept and reject regular expression", 2, &FileUtils::removeFilesFiltered($dir, '\.txt$', '(alpha|delta)'));
583 # - cleanup
584 &FileUtils::removeFiles($patha, $pathd);
585}
586## testRemoveFilesFiltered()
587
588
589## @function testRemoveFilesRecursive()
590#
591sub testRemoveFilesRecursive
592{
593 my ($dir) = @_;
594 # - setup
595 my $dira = $dir . '/alpha';
596 &FileUtils::makeDirectory($dira);
597 &FileUtils::filePutContents($dira . '/alpha.txt', 'alpha');
598 &FileUtils::filePutContents($dira . '/beta.txt', 'beta');
599 my $dirb = $dira . '/beta';
600 &FileUtils::makeDirectory($dirb);
601 &FileUtils::filePutContents($dirb . '/gamma.txt', 'gamma');
602 # - test
603 $pass_count += &testAction(\$test_count, "removeFilesRecursive() for directory containing files", 5, &FileUtils::removeFilesRecursive($dira));
604}
605## testRemoveFilesRecursive()
606
607
608## @function testTransferFiles()
609#
610sub testTransferFiles
611{
612 my ($dir, $mode, $local_dir) = @_;
613 my $verb = 'moving';
614 if ($mode eq 'copy')
615 {
616 $verb = 'copying';
617 }
618 # Move a single local file to a new local location
619 # - setup
620 my $apath = $dir . '/alpha.txt';
621 &FileUtils::filePutContents($apath,'alpha');
622 my $bpath = $dir . '/beta.txt';
623 my $sub_pass_count = 0;
624 # - subtests
625 if ($mode eq 'move')
626 {
627 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($apath, $bpath));
628 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($apath));
629 }
630 else
631 {
632 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($apath, $bpath));
633 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
634 }
635 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($bpath));
636
637 # - test
638 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " a single file", 3, $sub_pass_count);
639 # - cleanup
640 &FileUtils::removeFiles($bpath);
641
642 # Move multiple local files into a single local directory
643 # - setup
644 my $dpath = $dir . '/delta.txt';
645 &FileUtils::filePutContents($dpath, 'delta');
646 my $gpath = $dir . '/gamma.txt';
647 &FileUtils::filePutContents($gpath, 'gamma');
648 my $move_dir = $dir . '/movedir';
649 &FileUtils::makeDirectory($move_dir);
650 $sub_pass_count = 0;
651 # - subtests
652 if ($mode eq 'move')
653 {
654 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($dpath, $gpath, $move_dir));
655 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($dpath));
656 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($gpath));
657 }
658 else
659 {
660 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($dpath, $gpath, $move_dir));
661 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($dpath));
662 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($gpath));
663 }
664 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($move_dir . '/delta.txt'));
665 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($move_dir . '/gamma.txt'));
666 # - test
667 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a directory", 5, $sub_pass_count);
668 # - cleanup
669 if ($mode ne 'move')
670 {
671 &FileUtils::removeFiles($dpath, $gpath);
672 }
673
674 # Moving multiple files to file
675 if ($mode eq 'move')
676 {
677 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a file (should fail)", 0, &FileUtils::moveFiles($move_dir . '/delta.txt', $move_dir . '/gamma.txt', $bpath));
678 }
679 else
680 {
681 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a file (should fail)", 0, &FileUtils::copyFiles($move_dir . '/delta.txt', $move_dir . '/gamma.txt', $bpath));
682 }
683
684 # - cleanup
685 &FileUtils::removeFilesRecursive($move_dir);
686
687 if (defined $local_dir && -d $local_dir)
688 {
689 &FileUtils::filePutContents($apath, 'alpha');
690 my $zpath = $local_dir . '/fileutils-' . time() . '.txt';
691 # - init
692 $sub_pass_count = 0;
693 # - subtests
694 if ($mode eq 'move')
695 {
696 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($apath, $zpath));
697 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($apath));
698 }
699 else
700 {
701 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($apath, $zpath));
702 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
703 }
704 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($zpath));
705 # - test
706 $pass_count += &testAction(\$test_count, $mode . "Files() from non-local file to local one", 3, $sub_pass_count);
707
708 # If we are copying, we'll need to remove alpha, otherwise the round trip
709 # back to HDFS will fail
710 if ($mode eq 'copy')
711 {
712 &FileUtils::removeFiles($apath);
713 }
714
715 # - init
716 $sub_pass_count = 0;
717 # - subtests
718 if ($mode eq 'move')
719 {
720 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($zpath, $apath));
721 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($zpath));
722 }
723 else
724 {
725 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($zpath, $apath));
726 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($zpath));
727 }
728 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
729
730 # - test
731 $pass_count += &testAction(\$test_count, $mode . "Files() from local file to non-local one", 3, $sub_pass_count);
732 # - cleanup
733 if ($mode eq 'move')
734 {
735 &FileUtils::removeFiles($apath);
736 }
737 else
738 {
739 &FileUtils::removeFiles($apath, $zpath);
740 }
741 }
742 else
743 {
744 # Move a single non-local file to a new local location
745 $skip_count += &skipAction(\$test_count, $mode . "Files() from non-local file to local one");
746 # Move a single local file to a new non-local location
747 $skip_count += &skipAction(\$test_count, $mode . "Files() from local file to non-local one");
748 }
749}
750## testTransferFiles()
751
7521;
Note: See TracBrowser for help on using the repository browser.