source: gs2-extensions/parallel-building/trunk/src/bin/script/test_fileutils.pl@ 27436

Last change on this file since 27436 was 27436, checked in by jmt12, 11 years ago

Adding the actual script - rather than a symlink to my dropbox. doh
  • Property svn:executable set to *
File size: 28.4 KB
Line 
1#!/usr/bin/perl
2
3use strict;
4use warnings;
5no warnings 'once';
6
7# Configure
8my $user = 'jmt12';
9my $hadoop_path = '/mnt/sdb1/jmt12/gsdl2-svn/ext/parallel-building/packages/hadoop-1.1.0';
10my $redirect_errors = 0;
11my $display_errors = 0;
12
13my $test_localfs = 0;
14my $test_hdthriftfs = 1;
15my $test_hdfsshell = 0;
16
17# Globals
18my $base_path;
19my $test_count;
20my $pass_count;
21my $skip_count;
22my $start_time;
23my $total_time;
24my $error_messages;
25
26my $sample_str1 = "Neque porro quisquam est qui dolorem ipsum quia dolor sit amet,\nconsectetur,\nadipisci velit...\n";
27# This was going to be used for append (since Thrift has an append function)
28# but it turns out append is dependant upon HDFS being configured to allow
29# append - and mine isn't. Left here for those whom don't speak latin!
30my $sample_str2 = "[Translation]\nThere is no one who loves pain itself,\nwho seeks after it and wants to have it,\nsimply because it is pain...";
31
32# Requires setup.bash to have been sourced
33BEGIN
34{
35 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
36 die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
37 die "Not supported under windows\n" unless $ENV{'GSDLOS'} !~ /^windows$/i;
38 # Ensure Greenstone Perl locations are in INC
39 unshift (@INC, $ENV{'GSDLHOME'} . '/perllib');
40 unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/cpan');
41 if (defined $ENV{'GSDLEXTS'})
42 {
43 my @extensions = split(/:/, $ENV{'GSDLEXTS'});
44 foreach my $e (@extensions)
45 {
46 my $ext_prefix = $ENV{'GSDLHOME'} . '/ext/' . $e;
47 unshift (@INC, $ext_prefix . '/perllib');
48 unshift (@INC, $ext_prefix . '/perllib/cpan');
49 }
50 }
51}
52
53use Cwd;
54use DateTime;
55use Devel::Peek;
56
57use FileUtils;
58
59# populate globals
60$base_path = getcwd();
61
62print "\n";
63print "============================= Test FileUtils ============================\n";
64print "Test the FileUtils module to ensure it correctly manages local and HDFS\n";
65print "based files. HDFS support is provided by two different drivers: HDFSShell\n";
66print "uses repeated calls to the CLI Hadoop program, while HDThriftFS creates a\n";
67print "client socket connection to a running Thrift server.\n";
68print "=========================================================================\n";
69print "\n";
70
71if ($redirect_errors)
72{
73 open SAVEERR, ">&STDERR";
74 close STDERR;
75 open STDERR, ">", \$error_messages or die "What the hell?\n";
76}
77
78if ($test_localfs)
79{
80 $start_time = time();
81 print "A. Local File System\n";
82 $test_count = 0;
83 $pass_count = 0;
84 $skip_count = 0;
85 my $test_dir = '/tmp/fileutils';
86 # From this test on you are safe to assume the testing directory exists
87 &testMakeDirectory($test_dir);
88 &testFilePutContents($test_dir); # creates alpha.txt
89 &testFileGetContents($test_dir); # removes alpha.txt
90 &testFileExists($test_dir);
91 &testDirectoryExists($test_dir);
92 &testFilePermissions($test_dir);
93 &testFilenameConcatenate('/home/gsdl/collect/test/etc/collect.cfg', # target path
94 '/', 'home','gsdl','collect','test','etc','collect.cfg', #parts
95 );
96 &testOpenFileHandle($test_dir);
97 &testFileSize($test_dir);
98 &testModificationTime($test_dir);
99 &testDifferentFiles($test_dir);
100 &testReadDirectory($test_dir);
101 &testTransferFiles($test_dir, 'move');
102 &testTransferFiles($test_dir, 'copy');
103 &testLinkFile($test_dir, $base_path, 'soft');
104 &testLinkFile($test_dir, $base_path, 'hard');
105 $skip_count += &skipAction(\$test_count, "Binary file transfer test (roundtrip)");
106 &testRemoveFiles($test_dir);
107 &testRemoveFilesFiltered($test_dir);
108 &testRemoveFilesRecursive($test_dir);
109 print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
110 $total_time = time() - $start_time;
111 print "Time: " . $total_time . " seconds\n\n";
112 # Full cleanup before we start the next round of tests
113 &FileUtils::removeFilesRecursive($test_dir);
114}
115
116if ($test_hdthriftfs)
117{
118 $start_time = time();
119 print "B. HDFS using Thrift\n";
120 my $thrift_fileutils_path = 'HDThriftFS:///user/' . $user. '/fileutils';
121 $test_count = 0;
122 $pass_count = 0;
123 $skip_count = 0;
124 &testFilenameConcatenate('HDThriftFS:///home/gsdl/collect/test/etc/collect.cfg', # target path
125 'HDThriftFS://', 'home','gsdl','collect','test','etc','collect.cfg', #parts
126 );
127 &testMakeDirectory($thrift_fileutils_path);
128 &testFilePutContents($thrift_fileutils_path); # Leaves alpha.txt
129 &testFileGetContents($thrift_fileutils_path); # Removes alpha.txt
130 &testFileExists($thrift_fileutils_path);
131 &testDirectoryExists($thrift_fileutils_path);
132 &testFilePermissions($thrift_fileutils_path);
133 &testOpenFileHandle($thrift_fileutils_path);
134 &testFileSize($thrift_fileutils_path);
135 &testModificationTime($thrift_fileutils_path);
136 &testDifferentFiles($thrift_fileutils_path);
137 &testReadDirectory($thrift_fileutils_path);
138 &testTransferFiles($thrift_fileutils_path, 'move', '/tmp');
139 &testTransferFiles($thrift_fileutils_path, 'copy', '/tmp');
140 &testLinkFile($thrift_fileutils_path, '', 'soft');
141 &testLinkFile($thrift_fileutils_path, '', 'hard');
142 &testBinaryTransfer($thrift_fileutils_path, $base_path);
143 &testRemoveFiles($thrift_fileutils_path);
144 &testRemoveFilesFiltered($thrift_fileutils_path);
145 &testRemoveFilesRecursive($thrift_fileutils_path);
146 print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
147 $total_time = time() - $start_time;
148 print "Time: " . $total_time . " seconds\n\n";
149 # Cleanup
150 &FileUtils::removeFilesRecursive($thrift_fileutils_path);
151}
152
153if ($test_hdfsshell)
154{
155 $start_time = time();
156 print "C. HDFS using Shell\n";
157 my $hdfs_fileutils_path = 'HDFSShell:///user/' . $user. '/fileutils';
158 $test_count = 0;
159 $pass_count = 0;
160 $skip_count = 0;
161 &testFilenameConcatenate('HDFSShell:///home/gsdl/collect/test/etc/collect.cfg', # target path
162 'HDFSShell://', 'home','gsdl','collect','test','etc','collect.cfg', #parts
163 );
164 &testMakeDirectory($hdfs_fileutils_path);
165 &testFilePutContents($hdfs_fileutils_path); # Leaves alpha.txt
166 &testFileGetContents($hdfs_fileutils_path); # Removes alpha.txt
167 &testFileExists($hdfs_fileutils_path);
168 &testDirectoryExists($hdfs_fileutils_path);
169 &testFilePermissions($hdfs_fileutils_path);
170 &testOpenFileHandle($hdfs_fileutils_path);
171 &testFileSize($hdfs_fileutils_path);
172 &testModificationTime($hdfs_fileutils_path);
173 &testDifferentFiles($hdfs_fileutils_path);
174 &testReadDirectory($hdfs_fileutils_path);
175 &testTransferFiles($hdfs_fileutils_path, 'move', '/tmp');
176 &testTransferFiles($hdfs_fileutils_path, 'copy', '/tmp');
177 &testLinkFile($hdfs_fileutils_path, '', 'soft');
178 &testLinkFile($hdfs_fileutils_path, '', 'hard');
179 &testBinaryTransfer($hdfs_fileutils_path, $base_path);
180 &testRemoveFiles($hdfs_fileutils_path);
181 &testRemoveFilesFiltered($hdfs_fileutils_path);
182 &testRemoveFilesRecursive($hdfs_fileutils_path);
183 print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
184 $total_time = time() - $start_time;
185 print "Time: " . $total_time . " seconds\n\n";
186 # Cleanup
187 &FileUtils::removeFilesRecursive($hdfs_fileutils_path);
188}
189
190if ($redirect_errors)
191{
192 close STDERR;
193 open STDERR, ">&SAVEERR";
194 if ($display_errors)
195 {
196 print "Error Messages\n";
197 print $error_messages;
198 print "\n\n";
199 }
200}
201
202exit;
203
204# /** A test that doesn't actually test anything, but always skips. Used as a
205# * placeholder where certain tests aren't applicable.
206# */
207sub skipAction
208{
209 my ($test_count_ref, $description) = @_;
210 $$test_count_ref++;
211 printf("% 2d. %s: Skipped\n", $$test_count_ref, $description);
212 return 1;
213}
214
215sub subtestAction
216{
217 my ($expected_result, $actual_result, $debug) = @_;
218 if (defined $debug && $debug)
219 {
220 print "subtestAction(expected:$expected_result, actual:$actual_result)\n";
221 }
222 my $result = 'Fail';
223 if (defined $actual_result && (('' . $expected_result) eq ('' . $actual_result)))
224 {
225 $result = 'Pass';
226 }
227 return ('Pass' eq $result);
228}
229
230sub testAction
231{
232 my ($test_count_ref, $description, $expected_result, $actual_result) = @_;
233 $$test_count_ref++;
234 my $result = 'Fail';
235 if (defined $actual_result && (('' . $expected_result) eq ('' . $actual_result)))
236 {
237 $result = 'Pass';
238 }
239 else
240 {
241 $result .= ' (' . $expected_result . ' != ' . $actual_result . ')';
242 }
243 printf("% 2d. %s: %s\n", $$test_count_ref, $description, $result);
244 return ('Pass' eq $result);
245}
246
247################################################################################
248##### TESTS
249################################################################################
250
251## @function testBinaryTransfer()
252#
253sub testBinaryTransfer
254{
255 my ($dir, $base_dir) = @_;
256 # setup
257 my $local_image_path = $base_path . '/../../resources/aurora_australus.jpg';
258 my $thrift_copy_path = $dir . '/test_binary_transfer.jpg';
259 my $local_copy_path = '/tmp/test_binary_transfer.jpg';
260 # actions - roundtrip copy of files
261 &FileUtils::copyFiles($local_image_path, $thrift_copy_path);
262 &FileUtils::copyFiles($thrift_copy_path, $local_copy_path);
263 # test
264 my $result = `diff $local_image_path $local_copy_path`;
265 $pass_count += &testAction(\$test_count, "Binary file transfer test (roundtrip)", '', $result);
266 # cleanup
267 &FileUtils::removeFiles($thrift_copy_path, $local_copy_path);
268}
269## testBinaryTransfer()
270
271
272## @function testDifferentFiles
273#
274sub testDifferentFiles
275{
276 my ($dir) = @_;
277 my $patha = $dir . '/alpha.txt';
278 &FileUtils::filePutContents($patha, 'alpha');
279 my $pathb = $dir . '/beta.txt';
280 &FileUtils::filePutContents($pathb, 'beta');
281 $pass_count += &testAction(\$test_count, "differentFiles() comparing a file to itself", 0, &FileUtils::differentFiles($patha, $patha));
282 $pass_count += &testAction(\$test_count, "differentFiles() comparing a directory to itself", 0, &FileUtils::differentFiles($dir, $dir));
283 $pass_count += &testAction(\$test_count, "differentFiles() comparing different files", 1, &FileUtils::differentFiles($patha, $pathb));
284 $pass_count += &testAction(\$test_count, "differentFiles() comparing a file to a directory", 1, &FileUtils::differentFiles($patha, $dir));
285 # -cleanup
286 &FileUtils::removeFiles($patha, $pathb);
287}
288# /** testDifferentFiles() **/
289
290sub testDirectoryExists
291{
292 my ($patha, $pathb, $pathc, $pathd) = @_;
293 $pass_count += &testAction(\$test_count, "directoryExists() for existing directory", 1, &FileUtils::directoryExists($patha));
294 $pass_count += &testAction(\$test_count, "directoryExists() for existing file (should fail)", 0, &FileUtils::directoryExists($pathb));
295 $pass_count += &testAction(\$test_count, "directoryExists() for directory that doesn't exist (should fail)", 0, &FileUtils::directoryExists($pathc));
296 $pass_count += &testAction(\$test_count, "directoryExists() for file that doesn't exist (should fail)", 0, &FileUtils::directoryExists($pathd));
297}
298
299
300## @function testFileExists()
301#
302sub testFileExists
303{
304 my ($dir) = @_;
305 my $alpha_path = $dir . '/alpha.txt';
306 &FileUtils::filePutContents($alpha_path, 'alpha');
307 $pass_count += &testAction(\$test_count, "fileExists() for existing file", 1, &FileUtils::fileExists($alpha_path));
308 my $beta_path = $dir . '/beta.txt';
309 $pass_count += &testAction(\$test_count, "fileExists() for file that doesn't exist (should fail)", 0, &FileUtils::fileExists($beta_path));
310}
311## testFileExists()
312
313
314## @function testFileGetContents()
315#
316sub testFileGetContents
317{
318 my ($dir) = @_;
319 my $apath = $dir . '/alpha.txt';
320 $pass_count += &testAction(\$test_count, "fileGetContents() to read an entire file as a string", $sample_str1, &FileUtils::fileGetContents($apath));
321 &FileUtils::removeFiles($apath); # just have to hope this works
322}
323## testFileGetContents()
324
325
326## @function testFilePermissions()
327#
328sub testFilePermissions
329{
330 my ($dir) = @_;
331 my $patha = $dir . '/alpha.txt';
332 &FileUtils::filePutContents($patha, 'alpha');
333 $pass_count += &testAction(\$test_count, "canRead() testing read permission on a file I own", 1, &FileUtils::canRead($patha));
334 $pass_count += &testAction(\$test_count, "canRead() testing read permission on a file in my group", 1, &FileUtils::canRead($patha, 'johnsmith', 'supergroup'));
335 $pass_count += &testAction(\$test_count, "canRead() testing global access read permission on a file", 1, &FileUtils::canRead($patha, 'johnsmith', 'othergroup'));
336
337 $skip_count += &skipAction(\$test_count, "canWrite() testing write permission on a file");
338 $skip_count += &skipAction(\$test_count, "canExecute() testing execute permission on a file");
339 # - cleanup
340 &FileUtils::removeFiles($patha);
341}
342## testFilePermissions()
343
344
345## @function testFilePutContents()
346#
347sub testFilePutContents
348{
349 my ($dir) = @_;
350 my $apath = $dir . '/alpha.txt';
351 $pass_count += &testAction(\$test_count, "filePutContents() to write a string direct to a file", 1, &FileUtils::filePutContents($apath, $sample_str1));
352}
353## testFilePutContents()
354
355
356## @function testFileSize()
357#
358sub testFileSize
359{
360 my ($dir) = @_;
361 my $apath = $dir . '/alpha.txt';
362 my $astr = 'alpha';
363 &FileUtils::filePutContents($apath, $astr);
364 $pass_count += &testAction(\$test_count, "fileSize() to determine a file's size in bytes", length($astr), &FileUtils::fileSize($apath));
365 &FileUtils::removeFiles($apath);
366}
367## testFileSize()
368
369
370sub testFilenameConcatenate
371{
372 my $target = shift(@_);
373 $pass_count += &testAction(\$test_count, "filenameConcatenate() to combine file paths", $target, &FileUtils::filenameConcatenate(@_));
374}
375
376sub testLinkFile
377{
378 my ($dir, $base_path, $mode) = @_;
379 if (!&FileUtils::supportsSymbolicLink($dir))
380 {
381 &FileUtils::printWarning('Symbolic links not supported so files will be copied instead');
382 }
383
384 # - init
385 my $sub_pass_count = 0;
386 my $patha = $dir . '/alpha.txt';
387 &FileUtils::filePutContents($patha, 'alpha');
388 my $pathb = $dir . '/beta.txt';
389 # - subtests
390 if ($mode eq 'soft')
391 {
392 $sub_pass_count += &subtestAction(1, &FileUtils::softLink($patha, $pathb));
393 # If the driver doesn't support symbolic links, then the following function
394 # will actually end up triggering a '-e' - which is true even for hardlinks
395 $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
396 }
397 else
398 {
399 $sub_pass_count += &subtestAction(1, &FileUtils::hardLink($patha, $pathb));
400 if (&FileUtils::supportsSymbolicLink($pathb))
401 {
402 $sub_pass_count += &subtestAction(0, &FileUtils::isSymbolicLink($pathb));
403 }
404 # If the driver doesn't support symbolic links, then the following function
405 # will actually end up triggering a '-e' - which is true even for hardlinks
406 else
407 {
408 $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
409 }
410 }
411 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($patha));
412 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($pathb));
413 $sub_pass_count += &subtestAction('alpha', &FileUtils::fileGetContents($pathb));
414 # - test
415 $pass_count += &testAction(\$test_count, $mode . "Link() " . $mode . " symbolic linking a single file using relative path", 5, $sub_pass_count);
416 # - cleanup
417 &FileUtils::removeFiles($pathb);
418
419 # - init
420 if ($base_path ne '')
421 {
422 my $depth = split(/[\/\\]/, $base_path) - 1;
423 my $rel_prefix = '..';
424 for (my $i = 1; $i < $depth; $i++)
425 {
426 $rel_prefix .= '/..';
427 }
428 my $rel_patha = $rel_prefix . $patha;
429 my $rel_pathb = $rel_prefix . $pathb;
430 $sub_pass_count = 0;
431 # - subtests
432 if ($mode eq 'soft')
433 {
434 $sub_pass_count += &subtestAction(1, &FileUtils::softLink($rel_patha, $rel_pathb));
435 $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
436 }
437 else
438 {
439 $sub_pass_count += &subtestAction(1, &FileUtils::hardLink($rel_patha, $rel_pathb));
440 $sub_pass_count += &subtestAction(0, &FileUtils::isSymbolicLink($pathb));
441 }
442 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($patha));
443 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($pathb));
444 $sub_pass_count += &subtestAction('alpha', &FileUtils::fileGetContents($pathb));
445 # - test
446 $pass_count += &testAction(\$test_count, $mode . "Link() " . $mode . " symbolic linking a single file forcing absolute path", 5, $sub_pass_count);
447 # - cleanup
448 &FileUtils::removeFiles($pathb, $patha);
449 }
450 else
451 {
452 $skip_count += &skipAction(\$test_count, $mode . "Link() relative paths not supported in this filesystem");
453 }
454}
455
456sub testMakeDirectory
457{
458 my ($path) = @_;
459 $pass_count += &testAction(\$test_count, 'makeDirectory() to create a new directory', 1, &FileUtils::makeDirectory($path));
460 $pass_count += &testAction(\$test_count, 'makeDirectory() for an existing directory', 1, &FileUtils::makeDirectory($path));
461}
462
463sub testModificationTime
464{
465 my ($dir) = @_;
466 my $patha = $dir . '/alpha.txt';
467 my $exact_modification_time = time();
468 #rint STDERR "Exact Modification Time: $exact_modification_time\n";
469 &FileUtils::filePutContents($patha, 'Hello World');
470 my $file_modification_time = &FileUtils::modificationTime($patha);
471 #rint STDERR "File Modification Time: $file_modification_time\n";
472 # Try to find an exact match first
473 if ($exact_modification_time == $file_modification_time)
474 {
475 $pass_count += &testAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc (exact)', 1, 1);
476 }
477 # Failing that, we can try a rounded match (for file systems that
478 # don't store modification times down to the second i.e. HDFSShell)
479 else
480 {
481 $skip_count += &skipAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc (exact)');
482 my ($mod_second, $mod_minute, $mod_hour, $mod_day, $mod_month, $mod_year) = localtime($exact_modification_time);
483 $mod_year += 1900; # initially years since 1900, silly computer scientists
484 $mod_month += 1; # in the range [0..11], silly computer scientists
485 my $mod_datetime = DateTime->new(year => $mod_year,
486 month => $mod_month,
487 day => $mod_day,
488 hour => $mod_hour,
489 minute => $mod_minute,
490 second => 0,
491 time_zone => 'local');
492 my $start_modification_time = $mod_datetime->epoch();
493 #rint STDERR "Start Modification Time: $start_modification_time\n";
494 if ($start_modification_time == $file_modification_time)
495 {
496 $pass_count += &testAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc (rounded)', 1, 1);
497 }
498 # And finally we just accept any time that lies within the same minute
499 else
500 {
501 $skip_count += &skipAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc (rounded)');
502 $mod_datetime->add( minutes => 1 );
503 my $end_modification_time = $mod_datetime->epoch();
504 #rint STDERR "End Modification Time: $end_modification_time\n";
505 $pass_count += &testAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc (approximate)', 1, ($start_modification_time <= $file_modification_time && $file_modification_time <= $end_modification_time ? 1 : 0));
506 }
507 }
508 # cleanup
509 &FileUtils::removeFiles($patha);
510}
511
512sub testOpenFileHandle
513{
514 my ($dir) = @_;
515 my $path = $dir . '/alpha.txt';
516 my $dest_str;
517 my $file_handle;
518 # - writing
519 $pass_count += &testAction(\$test_count, "openFileHandle() to open a file handle for writing", 1, &FileUtils::openFileHandle($path, 'w', \$file_handle));
520 print $file_handle $sample_str1;
521 $pass_count += &testAction(\$test_count, "closeFileHandle() to close the writable file handle", 1, &FileUtils::closeFileHandle($path, \$file_handle));
522 # - reading
523 $pass_count += &testAction(\$test_count, "openFileHandle() to open a file handle for reading", 1, &FileUtils::openFileHandle($path, 'r', \$file_handle));
524 while (my $line = <$file_handle>)
525 {
526 $dest_str .= $line;
527 }
528 $pass_count += &testAction(\$test_count, "comparing string written to string read", $sample_str1, $dest_str);
529 $pass_count += &testAction(\$test_count, "closeFileHandle() to close the readable file handle", 1, &FileUtils::closeFileHandle($path, \$file_handle));
530 # - cleanup
531 &FileUtils::removeFiles($path);
532}
533
534sub testReadDirectory
535{
536 my ($dir) = @_;
537 my $apath = $dir . '/alpha.txt';
538 &FileUtils::filePutContents($apath, 'alpha');
539 my $bpath = $dir . '/beta.txt';
540 &FileUtils::filePutContents($bpath, 'beta');
541 my $dpath = $dir . '/delta.txt';
542 &FileUtils::filePutContents($dpath, 'delta');
543 my $file_count = 3;
544 # - read the directory, which will only contain the files we just put there
545 my @files = @{&FileUtils::readDirectory($dir)};
546 my $result_count = 0;
547 foreach my $file (@files)
548 {
549 if ($file !~ /^\./)
550 {
551 $result_count++;
552 }
553 }
554 $pass_count += &testAction(\$test_count, "readDirectory() listing files", $file_count, $result_count);
555 &FileUtils::removeFiles($apath, $bpath, $dpath);
556}
557
558## @function testRemoveFiles()
559#
560sub testRemoveFiles
561{
562 my ($dir) = @_;
563
564 my $patha = $dir . '/alpha.txt';
565 &FileUtils::filePutContents($patha, 'alpha');
566 my $pathb = $dir . '/beta.txt';
567 &FileUtils::softLink($patha, $pathb);
568 my $pathd = $dir . '/delta.txt';
569 &FileUtils::filePutContents($pathd, 'delta');
570 my $pathe = $dir . '/epsilon.txt';
571 &FileUtils::filePutContents($pathe, 'epsilon');
572 my $pathg = $dir . '/gamma.txt';
573 &FileUtils::filePutContents($pathg, 'gamma');
574 my $patht = $dir . '/tau.txt';
575 my $dirm = $dir .'/mu';
576 &FileUtils::makeDirectory($dirm);
577 # - tests
578 $pass_count += &testAction(\$test_count, "removeFiles() for existing file", 1, &FileUtils::removeFiles($patha));
579 $pass_count += &testAction(\$test_count, "removeFiles() for soft symbolic link", 1, &FileUtils::removeFiles($pathb));
580 $pass_count += &testAction(\$test_count, "removeFiles() for multiple existing files", 3, &FileUtils::removeFiles($pathd, $pathe, $pathg));
581 $pass_count += &testAction(\$test_count, "removeFiles() for file that doesn't exist (should fail)", 0, &FileUtils::removeFiles($patht));
582 $pass_count += &testAction(\$test_count, "removeFiles() for existing directory (should fail)", 0, &FileUtils::removeFiles($dirm));
583 # - cleanup
584 &FileUtils::removeFilesRecursive($dirm);
585}
586## testRemoveFiles()
587
588
589## @function testRemoveFilesFiltered()
590#
591sub testRemoveFilesFiltered
592{
593 my ($dir) = @_;
594 # - setup
595 my $patha = $dir . '/alpha.txt';
596 &FileUtils::filePutContents($patha, 'alpha');
597 &FileUtils::filePutContents($dir . '/beta.tmp','beta');
598 my $pathd = $dir . '/delta.txt';
599 &FileUtils::filePutContents($pathd, 'delta');
600 &FileUtils::filePutContents($dir . '/gamma.tmp','gamma');
601 &FileUtils::filePutContents($dir . '/epsilon.txt','epsilon');
602 &FileUtils::filePutContents($dir . '/eta.txt','eta');
603 # - tests
604 $pass_count += &testAction(\$test_count, "filteredRemove() with accept regular expression", 2, &FileUtils::removeFilesFiltered($dir, '\.tmp$', undef));
605 $pass_count += &testAction(\$test_count, "filteredRemove() with accept and reject regular expression", 2, &FileUtils::removeFilesFiltered($dir, '\.txt$', '(alpha|delta)'));
606 # - cleanup
607 &FileUtils::removeFiles($patha, $pathd);
608}
609## testRemoveFilesFiltered()
610
611
612## @function testRemoveFilesRecursive()
613#
614sub testRemoveFilesRecursive
615{
616 my ($dir) = @_;
617 # - setup
618 my $dira = $dir . '/alpha';
619 &FileUtils::makeDirectory($dira);
620 &FileUtils::filePutContents($dira . '/alpha.txt', 'alpha');
621 &FileUtils::filePutContents($dira . '/beta.txt', 'beta');
622 my $dirb = $dira . '/beta';
623 &FileUtils::makeDirectory($dirb);
624 &FileUtils::filePutContents($dirb . '/gamma.txt', 'gamma');
625 # - test
626 $pass_count += &testAction(\$test_count, "removeFilesRecursive() for directory containing files", 5, &FileUtils::removeFilesRecursive($dira));
627}
628## testRemoveFilesRecursive()
629
630
631## @function testTransferFiles()
632#
633sub testTransferFiles
634{
635 my ($dir, $mode, $local_dir) = @_;
636 my $verb = 'moving';
637 if ($mode eq 'copy')
638 {
639 $verb = 'copying';
640 }
641 # Move a single local file to a new local location
642 # - setup
643 my $apath = $dir . '/alpha.txt';
644 &FileUtils::filePutContents($apath,'alpha');
645 my $bpath = $dir . '/beta.txt';
646 my $sub_pass_count = 0;
647 # - subtests
648 if ($mode eq 'move')
649 {
650 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($apath, $bpath));
651 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($apath));
652 }
653 else
654 {
655 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($apath, $bpath));
656 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
657 }
658 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($bpath));
659
660 # - test
661 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " a single file", 3, $sub_pass_count);
662 # - cleanup
663 &FileUtils::removeFiles($bpath);
664
665 # Move multiple local files into a single local directory
666 # - setup
667 my $dpath = $dir . '/delta.txt';
668 &FileUtils::filePutContents($dpath, 'delta');
669 my $gpath = $dir . '/gamma.txt';
670 &FileUtils::filePutContents($gpath, 'gamma');
671 my $move_dir = $dir . '/movedir';
672 &FileUtils::makeDirectory($move_dir);
673 $sub_pass_count = 0;
674 # - subtests
675 if ($mode eq 'move')
676 {
677 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($dpath, $gpath, $move_dir));
678 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($dpath));
679 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($gpath));
680 }
681 else
682 {
683 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($dpath, $gpath, $move_dir));
684 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($dpath));
685 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($gpath));
686 }
687 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($move_dir . '/delta.txt'));
688 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($move_dir . '/gamma.txt'));
689 # - test
690 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a directory", 5, $sub_pass_count);
691 # - cleanup
692 if ($mode ne 'move')
693 {
694 &FileUtils::removeFiles($dpath, $gpath);
695 }
696
697 # Moving multiple files to file
698 if ($mode eq 'move')
699 {
700 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a file (should fail)", 0, &FileUtils::moveFiles($move_dir . '/delta.txt', $move_dir . '/gamma.txt', $bpath));
701 }
702 else
703 {
704 $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a file (should fail)", 0, &FileUtils::copyFiles($move_dir . '/delta.txt', $move_dir . '/gamma.txt', $bpath));
705 }
706
707 # - cleanup
708 &FileUtils::removeFilesRecursive($move_dir);
709
710 if (defined $local_dir && -d $local_dir)
711 {
712 &FileUtils::filePutContents($apath, 'alpha');
713 my $zpath = $local_dir . '/fileutils-' . time() . '.txt';
714 # - init
715 $sub_pass_count = 0;
716 # - subtests
717 if ($mode eq 'move')
718 {
719 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($apath, $zpath));
720 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($apath));
721 }
722 else
723 {
724 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($apath, $zpath));
725 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
726 }
727 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($zpath));
728 # - test
729 $pass_count += &testAction(\$test_count, $mode . "Files() from non-local file to local one", 3, $sub_pass_count);
730
731 # If we are copying, we'll need to remove alpha, otherwise the round trip
732 # back to HDFS will fail
733 if ($mode eq 'copy')
734 {
735 &FileUtils::removeFiles($apath);
736 }
737
738 # - init
739 $sub_pass_count = 0;
740 # - subtests
741 if ($mode eq 'move')
742 {
743 $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($zpath, $apath));
744 $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($zpath));
745 }
746 else
747 {
748 $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($zpath, $apath));
749 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($zpath));
750 }
751 $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
752
753 # - test
754 $pass_count += &testAction(\$test_count, $mode . "Files() from local file to non-local one", 3, $sub_pass_count);
755 # - cleanup
756 if ($mode eq 'move')
757 {
758 &FileUtils::removeFiles($apath);
759 }
760 else
761 {
762 &FileUtils::removeFiles($apath, $zpath);
763 }
764 }
765 else
766 {
767 # Move a single non-local file to a new local location
768 $skip_count += &skipAction(\$test_count, $mode . "Files() from non-local file to local one");
769 # Move a single local file to a new non-local location
770 $skip_count += &skipAction(\$test_count, $mode . "Files() from local file to non-local one");
771 }
772}
773## testTransferFiles()
774
7751;
Note: See TracBrowser for help on using the repository browser.