1 | #!/usr/bin/perl
|
---|
2 |
|
---|
3 | use strict;
|
---|
4 | use warnings;
|
---|
5 | no warnings 'once';
|
---|
6 |
|
---|
7 | # Configure
|
---|
8 | my $user = 'jmt12';
|
---|
9 | my $redirect_errors = 0;
|
---|
10 | my $display_errors = 0;
|
---|
11 |
|
---|
12 | my $test_localfs = 0;
|
---|
13 | my $test_hdthriftfs = 1;
|
---|
14 | my $test_hdfsshell = 0;
|
---|
15 |
|
---|
16 | # Globals
|
---|
17 | my $base_path;
|
---|
18 | my $test_count;
|
---|
19 | my $pass_count;
|
---|
20 | my $skip_count;
|
---|
21 | my $start_time;
|
---|
22 | my $total_time;
|
---|
23 | my $error_messages;
|
---|
24 |
|
---|
25 | my $sample_str1 = "Neque porro quisquam est qui dolorem ipsum quia dolor sit amet,\nconsectetur,\nadipisci velit...\n";
|
---|
26 | # This was going to be used for append (since Thrift has an append function)
|
---|
27 | # but it turns out append is dependant upon HDFS being configured to allow
|
---|
28 | # append - and mine isn't. Left here for those whom don't speak latin!
|
---|
29 | my $sample_str2 = "[Translation]\nThere is no one who loves pain itself,\nwho seeks after it and wants to have it,\nsimply because it is pain...";
|
---|
30 |
|
---|
31 | # Requires setup.bash to have been sourced
|
---|
32 | BEGIN
|
---|
33 | {
|
---|
34 | die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
|
---|
35 | die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
|
---|
36 | die "Not supported under windows\n" unless $ENV{'GSDLOS'} !~ /^windows$/i;
|
---|
37 | # Ensure Greenstone Perl locations are in INC
|
---|
38 | unshift (@INC, $ENV{'GSDLHOME'} . '/perllib');
|
---|
39 | unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/cpan');
|
---|
40 | if (defined $ENV{'GSDLEXTS'})
|
---|
41 | {
|
---|
42 | my @extensions = split(/:/, $ENV{'GSDLEXTS'});
|
---|
43 | foreach my $e (@extensions)
|
---|
44 | {
|
---|
45 | my $ext_prefix = $ENV{'GSDLHOME'} . '/ext/' . $e;
|
---|
46 | unshift (@INC, $ext_prefix . '/perllib');
|
---|
47 | unshift (@INC, $ext_prefix . '/perllib/cpan');
|
---|
48 | }
|
---|
49 | }
|
---|
50 |
|
---|
51 | # Manually installed CPAN package in GEXT*INSTALL
|
---|
52 | # - parse up version number
|
---|
53 | my ($major, $minor, $revision) = $] =~ /(\d+)\.(\d\d\d)(\d\d\d)/;
|
---|
54 | # - get rid of leading zeros by making them integers
|
---|
55 | $major += 0;
|
---|
56 | $minor += 0;
|
---|
57 | $revision += 0;
|
---|
58 | # - and add to Perl's path
|
---|
59 | unshift (@INC, $ENV{'GEXTPARALLELBUILDING_INSTALLED'} . '/lib/perl/' . $major . '.' . $minor . '.' . $revision);
|
---|
60 | }
|
---|
61 |
|
---|
62 | use Cwd;
|
---|
63 | use Devel::Peek;
|
---|
64 |
|
---|
65 | use FileUtils;
|
---|
66 |
|
---|
67 | # populate globals
|
---|
68 | $base_path = $ENV{'GEXTPARALLELBUILDING'};
|
---|
69 |
|
---|
70 | # SPECIAL TESTS
|
---|
71 | if (defined $ARGV[0])
|
---|
72 | {
|
---|
73 | if ($ARGV[0] eq "-buffersize")
|
---|
74 | {
|
---|
75 | print STDERR "Exploring the effect of ThriftBuffer size on transfer speed\n";
|
---|
76 | my $filename = 'aurora_australus.jpg';
|
---|
77 | if (defined $ARGV[1])
|
---|
78 | {
|
---|
79 | $filename = $ARGV[1];
|
---|
80 | }
|
---|
81 | my $local_path = $base_path . '/resources/' . $filename;
|
---|
82 | my $thrift_path = 'HDThriftFS:///user/' . $user. '/' . $filename;
|
---|
83 | my $tmp_path = '/tmp/' . $filename;
|
---|
84 | print STDERR " - copying " . $filename . " (" . &FileUtils::fileSize($local_path) . " bytes)\n";
|
---|
85 | $start_time = time();
|
---|
86 | print STDERR " - copy to Thrift... ";
|
---|
87 | &FileUtils::copyFiles($local_path, $thrift_path);
|
---|
88 | print STDERR "Done in " . (time() - $start_time) . " seconds\n";
|
---|
89 | $start_time = time();
|
---|
90 | print STDERR " - copy from Thrift...";
|
---|
91 | &FileUtils::copyFiles($thrift_path, $tmp_path);
|
---|
92 | print STDERR "Done in " . (time() - $start_time) . " seconds\n";
|
---|
93 | my $result = `diff "$local_path" "$tmp_path"`;
|
---|
94 | $test_count = 0;
|
---|
95 | &testAction(\$test_count, "Comparing binary files (roundtrip)", '', $result);
|
---|
96 | &FileUtils::removeFiles($thrift_path);
|
---|
97 | &FileUtils::removeFiles($tmp_path);
|
---|
98 | exit;
|
---|
99 | }
|
---|
100 | }
|
---|
101 |
|
---|
102 | print "\n";
|
---|
103 | print "============================= Test FileUtils ============================\n";
|
---|
104 | print "Test the FileUtils module to ensure it correctly manages local and HDFS\n";
|
---|
105 | print "based files. HDFS support is provided by two different drivers: HDFSShell\n";
|
---|
106 | print "uses repeated calls to the CLI Hadoop program, while HDThriftFS creates a\n";
|
---|
107 | print "client socket connection to a running Thrift server.\n";
|
---|
108 | print "=========================================================================\n";
|
---|
109 | print "\n";
|
---|
110 |
|
---|
111 | if ($redirect_errors)
|
---|
112 | {
|
---|
113 | open SAVEERR, ">&STDERR";
|
---|
114 | close STDERR;
|
---|
115 | open STDERR, ">", \$error_messages or die "What the hell?\n";
|
---|
116 | }
|
---|
117 |
|
---|
118 | if ($test_localfs)
|
---|
119 | {
|
---|
120 | $start_time = time();
|
---|
121 | print "A. Local File System\n";
|
---|
122 | $test_count = 0;
|
---|
123 | $pass_count = 0;
|
---|
124 | $skip_count = 0;
|
---|
125 | my $test_dir = '/tmp/fileutils';
|
---|
126 | # From this test on you are safe to assume the testing directory exists
|
---|
127 | &testMakeDirectory($test_dir);
|
---|
128 | &testFilePutContents($test_dir); # creates alpha.txt
|
---|
129 | &testFileGetContents($test_dir); # removes alpha.txt
|
---|
130 | &testFileExists($test_dir);
|
---|
131 | &testDirectoryExists($test_dir);
|
---|
132 | &testFilePermissions($test_dir);
|
---|
133 | &testFilenameConcatenate('/home/gsdl/collect/test/etc/collect.cfg', # target path
|
---|
134 | '/', 'home','gsdl','collect','test','etc','collect.cfg', #parts
|
---|
135 | );
|
---|
136 | &testOpenFileHandle($test_dir);
|
---|
137 | &testFileSize($test_dir);
|
---|
138 | &testModificationTime($test_dir);
|
---|
139 | &testDifferentFiles($test_dir);
|
---|
140 | &testReadDirectory($test_dir);
|
---|
141 | &testTransferFiles($test_dir, 'move');
|
---|
142 | &testTransferFiles($test_dir, 'copy');
|
---|
143 | &testLinkFile($test_dir, $base_path, 'soft');
|
---|
144 | &testLinkFile($test_dir, $base_path, 'hard');
|
---|
145 | $skip_count += &skipAction(\$test_count, "Binary file transfer test (roundtrip)");
|
---|
146 | &testRemoveFiles($test_dir);
|
---|
147 | &testRemoveFilesFiltered($test_dir);
|
---|
148 | &testRemoveFilesRecursive($test_dir);
|
---|
149 | print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
|
---|
150 | $total_time = time() - $start_time;
|
---|
151 | print "Time: " . $total_time . " seconds\n\n";
|
---|
152 | # Full cleanup before we start the next round of tests
|
---|
153 | &FileUtils::removeFilesRecursive($test_dir);
|
---|
154 | }
|
---|
155 |
|
---|
156 | if ($test_hdthriftfs)
|
---|
157 | {
|
---|
158 | $start_time = time();
|
---|
159 | print "B. HDFS using Thrift\n";
|
---|
160 | my $thrift_fileutils_path = 'HDThriftFS:///user/' . $user. '/fileutils';
|
---|
161 | $test_count = 0;
|
---|
162 | $pass_count = 0;
|
---|
163 | $skip_count = 0;
|
---|
164 | &testFilenameConcatenate('HDThriftFS:///home/gsdl/collect/test/etc/collect.cfg', # target path
|
---|
165 | 'HDThriftFS://', 'home','gsdl','collect','test','etc','collect.cfg', #parts
|
---|
166 | );
|
---|
167 | &testMakeDirectory($thrift_fileutils_path);
|
---|
168 | &testFilePutContents($thrift_fileutils_path); # Leaves alpha.txt
|
---|
169 | &testFileGetContents($thrift_fileutils_path); # Removes alpha.txt
|
---|
170 | &testFileExists($thrift_fileutils_path);
|
---|
171 | &testDirectoryExists($thrift_fileutils_path);
|
---|
172 | &testFilePermissions($thrift_fileutils_path);
|
---|
173 | &testOpenFileHandle($thrift_fileutils_path);
|
---|
174 | &testFileSize($thrift_fileutils_path);
|
---|
175 | &testModificationTime($thrift_fileutils_path);
|
---|
176 | &testDifferentFiles($thrift_fileutils_path);
|
---|
177 | &testReadDirectory($thrift_fileutils_path);
|
---|
178 | &testTransferFiles($thrift_fileutils_path, 'move', '/tmp');
|
---|
179 | &testTransferFiles($thrift_fileutils_path, 'copy', '/tmp');
|
---|
180 | &testLinkFile($thrift_fileutils_path, '', 'soft');
|
---|
181 | &testLinkFile($thrift_fileutils_path, '', 'hard');
|
---|
182 | &testBinaryTransfer($thrift_fileutils_path, $base_path);
|
---|
183 | &testRemoveFiles($thrift_fileutils_path);
|
---|
184 | &testRemoveFilesFiltered($thrift_fileutils_path);
|
---|
185 | &testRemoveFilesRecursive($thrift_fileutils_path);
|
---|
186 | print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
|
---|
187 | $total_time = time() - $start_time;
|
---|
188 | print "Time: " . $total_time . " seconds\n\n";
|
---|
189 | # Cleanup
|
---|
190 | &FileUtils::removeFilesRecursive($thrift_fileutils_path);
|
---|
191 | }
|
---|
192 |
|
---|
193 | if ($test_hdfsshell)
|
---|
194 | {
|
---|
195 | $start_time = time();
|
---|
196 | print "C. HDFS using Shell\n";
|
---|
197 | my $hdfs_fileutils_path = 'HDFSShell:///user/' . $user. '/fileutils';
|
---|
198 | $test_count = 0;
|
---|
199 | $pass_count = 0;
|
---|
200 | $skip_count = 0;
|
---|
201 | &testFilenameConcatenate('HDFSShell:///home/gsdl/collect/test/etc/collect.cfg', # target path
|
---|
202 | 'HDFSShell://', 'home','gsdl','collect','test','etc','collect.cfg', #parts
|
---|
203 | );
|
---|
204 | &testMakeDirectory($hdfs_fileutils_path);
|
---|
205 | &testFilePutContents($hdfs_fileutils_path); # Leaves alpha.txt
|
---|
206 | &testFileGetContents($hdfs_fileutils_path); # Removes alpha.txt
|
---|
207 | &testFileExists($hdfs_fileutils_path);
|
---|
208 | &testDirectoryExists($hdfs_fileutils_path);
|
---|
209 | &testFilePermissions($hdfs_fileutils_path);
|
---|
210 | &testOpenFileHandle($hdfs_fileutils_path);
|
---|
211 | &testFileSize($hdfs_fileutils_path);
|
---|
212 | &testModificationTime($hdfs_fileutils_path);
|
---|
213 | &testDifferentFiles($hdfs_fileutils_path);
|
---|
214 | &testReadDirectory($hdfs_fileutils_path);
|
---|
215 | &testTransferFiles($hdfs_fileutils_path, 'move', '/tmp');
|
---|
216 | &testTransferFiles($hdfs_fileutils_path, 'copy', '/tmp');
|
---|
217 | &testLinkFile($hdfs_fileutils_path, '', 'soft');
|
---|
218 | &testLinkFile($hdfs_fileutils_path, '', 'hard');
|
---|
219 | &testBinaryTransfer($hdfs_fileutils_path, $base_path);
|
---|
220 | &testRemoveFiles($hdfs_fileutils_path);
|
---|
221 | &testRemoveFilesFiltered($hdfs_fileutils_path);
|
---|
222 | &testRemoveFilesRecursive($hdfs_fileutils_path);
|
---|
223 | print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
|
---|
224 | $total_time = time() - $start_time;
|
---|
225 | print "Time: " . $total_time . " seconds\n\n";
|
---|
226 | # Cleanup
|
---|
227 | &FileUtils::removeFilesRecursive($hdfs_fileutils_path);
|
---|
228 | }
|
---|
229 |
|
---|
230 | if ($redirect_errors)
|
---|
231 | {
|
---|
232 | close STDERR;
|
---|
233 | open STDERR, ">&SAVEERR";
|
---|
234 | if ($display_errors)
|
---|
235 | {
|
---|
236 | print "Error Messages\n";
|
---|
237 | print $error_messages;
|
---|
238 | print "\n\n";
|
---|
239 | }
|
---|
240 | }
|
---|
241 |
|
---|
242 | exit;
|
---|
243 |
|
---|
244 | # /** A test that doesn't actually test anything, but always skips. Used as a
|
---|
245 | # * placeholder where certain tests aren't applicable.
|
---|
246 | # */
|
---|
247 | sub skipAction
|
---|
248 | {
|
---|
249 | my ($test_count_ref, $description) = @_;
|
---|
250 | $$test_count_ref++;
|
---|
251 | printf("% 2d. %s: Skipped\n", $$test_count_ref, $description);
|
---|
252 | return 1;
|
---|
253 | }
|
---|
254 |
|
---|
255 | sub subtestAction
|
---|
256 | {
|
---|
257 | my ($expected_result, $actual_result, $debug) = @_;
|
---|
258 | if (defined $debug && $debug)
|
---|
259 | {
|
---|
260 | print "subtestAction(expected:$expected_result, actual:$actual_result)\n";
|
---|
261 | }
|
---|
262 | my $result = 'Fail';
|
---|
263 | if (defined $actual_result && (('' . $expected_result) eq ('' . $actual_result)))
|
---|
264 | {
|
---|
265 | $result = 'Pass';
|
---|
266 | }
|
---|
267 | return ('Pass' eq $result);
|
---|
268 | }
|
---|
269 |
|
---|
270 | sub testAction
|
---|
271 | {
|
---|
272 | my ($test_count_ref, $description, $expected_result, $actual_result) = @_;
|
---|
273 | $$test_count_ref++;
|
---|
274 | my $result = 'Fail';
|
---|
275 | if (defined $actual_result && (('' . $expected_result) eq ('' . $actual_result)))
|
---|
276 | {
|
---|
277 | $result = 'Pass';
|
---|
278 | }
|
---|
279 | else
|
---|
280 | {
|
---|
281 | $result .= ' (' . $expected_result . ' != ' . $actual_result . ')';
|
---|
282 | }
|
---|
283 | printf("% 2d. %s: %s\n", $$test_count_ref, $description, $result);
|
---|
284 | return ('Pass' eq $result);
|
---|
285 | }
|
---|
286 |
|
---|
287 | ################################################################################
|
---|
288 | ##### TESTS
|
---|
289 | ################################################################################
|
---|
290 |
|
---|
291 | ## @function testBinaryTransfer()
|
---|
292 | #
|
---|
293 | sub testBinaryTransfer
|
---|
294 | {
|
---|
295 | my ($dir, $base_dir, $file) = @_;
|
---|
296 | if (!defined $file)
|
---|
297 | {
|
---|
298 | $file = 'aurora_australus.jpg';
|
---|
299 | }
|
---|
300 | # setup
|
---|
301 | my ($extension) = $file =~ /\.([^\.]+)$/;
|
---|
302 | my $local_image_path = $base_path . '/resources/' . $file;
|
---|
303 | my $thrift_copy_path = $dir . '/test_binary_transfer.' . $extension;
|
---|
304 | my $local_copy_path = '/tmp/test_binary_transfer.' . $extension;
|
---|
305 | # actions - roundtrip copy of files
|
---|
306 | &FileUtils::copyFiles($local_image_path, $thrift_copy_path);
|
---|
307 | &FileUtils::copyFiles($thrift_copy_path, $local_copy_path);
|
---|
308 | # test
|
---|
309 | my $result = `diff $local_image_path $local_copy_path`;
|
---|
310 | $pass_count += &testAction(\$test_count, "Binary file transfer test (roundtrip)", '', $result);
|
---|
311 | # cleanup
|
---|
312 | &FileUtils::removeFiles($thrift_copy_path, $local_copy_path);
|
---|
313 | }
|
---|
314 | ## testBinaryTransfer()
|
---|
315 |
|
---|
316 |
|
---|
317 | ## @function testDifferentFiles
|
---|
318 | #
|
---|
319 | sub testDifferentFiles
|
---|
320 | {
|
---|
321 | my ($dir) = @_;
|
---|
322 | my $patha = $dir . '/alpha.txt';
|
---|
323 | &FileUtils::filePutContents($patha, 'alpha');
|
---|
324 | my $pathb = $dir . '/beta.txt';
|
---|
325 | &FileUtils::filePutContents($pathb, 'beta');
|
---|
326 | $pass_count += &testAction(\$test_count, "differentFiles() comparing a file to itself", 0, &FileUtils::differentFiles($patha, $patha));
|
---|
327 | $pass_count += &testAction(\$test_count, "differentFiles() comparing a directory to itself", 0, &FileUtils::differentFiles($dir, $dir));
|
---|
328 | $pass_count += &testAction(\$test_count, "differentFiles() comparing different files", 1, &FileUtils::differentFiles($patha, $pathb));
|
---|
329 | $pass_count += &testAction(\$test_count, "differentFiles() comparing a file to a directory", 1, &FileUtils::differentFiles($patha, $dir));
|
---|
330 | # -cleanup
|
---|
331 | &FileUtils::removeFiles($patha, $pathb);
|
---|
332 | }
|
---|
333 | # /** testDifferentFiles() **/
|
---|
334 |
|
---|
335 | sub testDirectoryExists
|
---|
336 | {
|
---|
337 | my ($patha, $pathb, $pathc, $pathd) = @_;
|
---|
338 | $pass_count += &testAction(\$test_count, "directoryExists() for existing directory", 1, &FileUtils::directoryExists($patha));
|
---|
339 | $pass_count += &testAction(\$test_count, "directoryExists() for existing file (should fail)", 0, &FileUtils::directoryExists($pathb));
|
---|
340 | $pass_count += &testAction(\$test_count, "directoryExists() for directory that doesn't exist (should fail)", 0, &FileUtils::directoryExists($pathc));
|
---|
341 | $pass_count += &testAction(\$test_count, "directoryExists() for file that doesn't exist (should fail)", 0, &FileUtils::directoryExists($pathd));
|
---|
342 | }
|
---|
343 |
|
---|
344 |
|
---|
345 | ## @function testFileExists()
|
---|
346 | #
|
---|
347 | sub testFileExists
|
---|
348 | {
|
---|
349 | my ($dir) = @_;
|
---|
350 | my $alpha_path = $dir . '/alpha.txt';
|
---|
351 | &FileUtils::filePutContents($alpha_path, 'alpha');
|
---|
352 | $pass_count += &testAction(\$test_count, "fileExists() for existing file", 1, &FileUtils::fileExists($alpha_path));
|
---|
353 | my $beta_path = $dir . '/beta.txt';
|
---|
354 | $pass_count += &testAction(\$test_count, "fileExists() for file that doesn't exist (should fail)", 0, &FileUtils::fileExists($beta_path));
|
---|
355 | }
|
---|
356 | ## testFileExists()
|
---|
357 |
|
---|
358 |
|
---|
359 | ## @function testFileGetContents()
|
---|
360 | #
|
---|
361 | sub testFileGetContents
|
---|
362 | {
|
---|
363 | my ($dir) = @_;
|
---|
364 | my $apath = $dir . '/alpha.txt';
|
---|
365 | $pass_count += &testAction(\$test_count, "fileGetContents() to read an entire file as a string", $sample_str1, &FileUtils::fileGetContents($apath));
|
---|
366 | &FileUtils::removeFiles($apath); # just have to hope this works
|
---|
367 | }
|
---|
368 | ## testFileGetContents()
|
---|
369 |
|
---|
370 |
|
---|
371 | ## @function testFilePermissions()
|
---|
372 | #
|
---|
373 | sub testFilePermissions
|
---|
374 | {
|
---|
375 | my ($dir) = @_;
|
---|
376 | my $patha = $dir . '/alpha.txt';
|
---|
377 | &FileUtils::filePutContents($patha, 'alpha');
|
---|
378 | $pass_count += &testAction(\$test_count, "canRead() testing read permission on a file I own", 1, &FileUtils::canRead($patha));
|
---|
379 | $pass_count += &testAction(\$test_count, "canRead() testing read permission on a file in my group", 1, &FileUtils::canRead($patha, 'johnsmith', 'supergroup'));
|
---|
380 | $pass_count += &testAction(\$test_count, "canRead() testing global access read permission on a file", 1, &FileUtils::canRead($patha, 'johnsmith', 'othergroup'));
|
---|
381 |
|
---|
382 | $skip_count += &skipAction(\$test_count, "canWrite() testing write permission on a file");
|
---|
383 | $skip_count += &skipAction(\$test_count, "canExecute() testing execute permission on a file");
|
---|
384 | # - cleanup
|
---|
385 | &FileUtils::removeFiles($patha);
|
---|
386 | }
|
---|
387 | ## testFilePermissions()
|
---|
388 |
|
---|
389 |
|
---|
390 | ## @function testFilePutContents()
|
---|
391 | #
|
---|
392 | sub testFilePutContents
|
---|
393 | {
|
---|
394 | my ($dir) = @_;
|
---|
395 | my $apath = $dir . '/alpha.txt';
|
---|
396 | $pass_count += &testAction(\$test_count, "filePutContents() to write a string direct to a file", 1, &FileUtils::filePutContents($apath, $sample_str1));
|
---|
397 | }
|
---|
398 | ## testFilePutContents()
|
---|
399 |
|
---|
400 |
|
---|
401 | ## @function testFileSize()
|
---|
402 | #
|
---|
403 | sub testFileSize
|
---|
404 | {
|
---|
405 | my ($dir) = @_;
|
---|
406 | my $apath = $dir . '/alpha.txt';
|
---|
407 | my $astr = 'alpha';
|
---|
408 | &FileUtils::filePutContents($apath, $astr);
|
---|
409 | $pass_count += &testAction(\$test_count, "fileSize() to determine a file's size in bytes", length($astr), &FileUtils::fileSize($apath));
|
---|
410 | &FileUtils::removeFiles($apath);
|
---|
411 | }
|
---|
412 | ## testFileSize()
|
---|
413 |
|
---|
414 |
|
---|
415 | sub testFilenameConcatenate
|
---|
416 | {
|
---|
417 | my $target = shift(@_);
|
---|
418 | $pass_count += &testAction(\$test_count, "filenameConcatenate() to combine file paths", $target, &FileUtils::filenameConcatenate(@_));
|
---|
419 | }
|
---|
420 |
|
---|
421 | sub testLinkFile
|
---|
422 | {
|
---|
423 | my ($dir, $base_path, $mode) = @_;
|
---|
424 | if (!&FileUtils::supportsSymbolicLink($dir))
|
---|
425 | {
|
---|
426 | &FileUtils::printWarning('Symbolic links not supported so files will be copied instead');
|
---|
427 | }
|
---|
428 |
|
---|
429 | # - init
|
---|
430 | my $sub_pass_count = 0;
|
---|
431 | my $patha = $dir . '/alpha.txt';
|
---|
432 | &FileUtils::filePutContents($patha, 'alpha');
|
---|
433 | my $pathb = $dir . '/beta.txt';
|
---|
434 | # - subtests
|
---|
435 | if ($mode eq 'soft')
|
---|
436 | {
|
---|
437 | $sub_pass_count += &subtestAction(1, &FileUtils::softLink($patha, $pathb));
|
---|
438 | # If the driver doesn't support symbolic links, then the following function
|
---|
439 | # will actually end up triggering a '-e' - which is true even for hardlinks
|
---|
440 | $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
|
---|
441 | }
|
---|
442 | else
|
---|
443 | {
|
---|
444 | $sub_pass_count += &subtestAction(1, &FileUtils::hardLink($patha, $pathb));
|
---|
445 | if (&FileUtils::supportsSymbolicLink($pathb))
|
---|
446 | {
|
---|
447 | $sub_pass_count += &subtestAction(0, &FileUtils::isSymbolicLink($pathb));
|
---|
448 | }
|
---|
449 | # If the driver doesn't support symbolic links, then the following function
|
---|
450 | # will actually end up triggering a '-e' - which is true even for hardlinks
|
---|
451 | else
|
---|
452 | {
|
---|
453 | $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
|
---|
454 | }
|
---|
455 | }
|
---|
456 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($patha));
|
---|
457 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($pathb));
|
---|
458 | $sub_pass_count += &subtestAction('alpha', &FileUtils::fileGetContents($pathb));
|
---|
459 | # - test
|
---|
460 | $pass_count += &testAction(\$test_count, $mode . "Link() " . $mode . " symbolic linking a single file using relative path", 5, $sub_pass_count);
|
---|
461 | # - cleanup
|
---|
462 | &FileUtils::removeFiles($pathb);
|
---|
463 |
|
---|
464 | # - init
|
---|
465 | if ($base_path ne '')
|
---|
466 | {
|
---|
467 | my $depth = split(/[\/\\]/, $base_path) - 1;
|
---|
468 | my $rel_prefix = '..';
|
---|
469 | for (my $i = 1; $i < $depth; $i++)
|
---|
470 | {
|
---|
471 | $rel_prefix .= '/..';
|
---|
472 | }
|
---|
473 | my $rel_patha = $rel_prefix . $patha;
|
---|
474 | my $rel_pathb = $rel_prefix . $pathb;
|
---|
475 | $sub_pass_count = 0;
|
---|
476 | # - subtests
|
---|
477 | if ($mode eq 'soft')
|
---|
478 | {
|
---|
479 | $sub_pass_count += &subtestAction(1, &FileUtils::softLink($rel_patha, $rel_pathb));
|
---|
480 | $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
|
---|
481 | }
|
---|
482 | else
|
---|
483 | {
|
---|
484 | $sub_pass_count += &subtestAction(1, &FileUtils::hardLink($rel_patha, $rel_pathb));
|
---|
485 | $sub_pass_count += &subtestAction(0, &FileUtils::isSymbolicLink($pathb));
|
---|
486 | }
|
---|
487 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($patha));
|
---|
488 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($pathb));
|
---|
489 | $sub_pass_count += &subtestAction('alpha', &FileUtils::fileGetContents($pathb));
|
---|
490 | # - test
|
---|
491 | $pass_count += &testAction(\$test_count, $mode . "Link() " . $mode . " symbolic linking a single file forcing absolute path", 5, $sub_pass_count);
|
---|
492 | # - cleanup
|
---|
493 | &FileUtils::removeFiles($pathb, $patha);
|
---|
494 | }
|
---|
495 | else
|
---|
496 | {
|
---|
497 | $skip_count += &skipAction(\$test_count, $mode . "Link() relative paths not supported in this filesystem");
|
---|
498 | }
|
---|
499 | }
|
---|
500 |
|
---|
501 | sub testMakeDirectory
|
---|
502 | {
|
---|
503 | my ($path) = @_;
|
---|
504 | $pass_count += &testAction(\$test_count, 'makeDirectory() to create a new directory', 1, &FileUtils::makeDirectory($path));
|
---|
505 | $pass_count += &testAction(\$test_count, 'makeDirectory() for an existing directory', 1, &FileUtils::makeDirectory($path));
|
---|
506 |
|
---|
507 | my $multiple_dirs_path = $path . '/foo/bar/wibble';
|
---|
508 | $pass_count += &testAction(\$test_count, 'makeAllDirectories() to create several nested directories', 1, &FileUtils::makeAllDirectories($multiple_dirs_path));
|
---|
509 | &FileUtils::removeFilesRecursive($path . '/foo');
|
---|
510 | }
|
---|
511 |
|
---|
512 | sub testModificationTime
|
---|
513 | {
|
---|
514 | my ($dir) = @_;
|
---|
515 | # setup
|
---|
516 | my $patha = $dir . '/alpha.txt';
|
---|
517 | my $modification_time = time();
|
---|
518 | &FileUtils::filePutContents($patha, 'Hello World');
|
---|
519 | my $file_modification_time = &FileUtils::modificationTime($patha);
|
---|
520 | # test
|
---|
521 | $pass_count += &testAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc', $modification_time, $file_modification_time);
|
---|
522 | # cleanup
|
---|
523 | &FileUtils::removeFiles($patha);
|
---|
524 | }
|
---|
525 |
|
---|
526 | sub testOpenFileHandle
|
---|
527 | {
|
---|
528 | my ($dir) = @_;
|
---|
529 | my $path = $dir . '/alpha.txt';
|
---|
530 | my $dest_str;
|
---|
531 | my $file_handle;
|
---|
532 | # - writing
|
---|
533 | $pass_count += &testAction(\$test_count, "openFileHandle() to open a file handle for writing", 1, &FileUtils::openFileHandle($path, 'w', \$file_handle));
|
---|
534 | print $file_handle $sample_str1;
|
---|
535 | $pass_count += &testAction(\$test_count, "closeFileHandle() to close the writable file handle", 1, &FileUtils::closeFileHandle($path, \$file_handle));
|
---|
536 | # - reading
|
---|
537 | $pass_count += &testAction(\$test_count, "openFileHandle() to open a file handle for reading", 1, &FileUtils::openFileHandle($path, 'r', \$file_handle));
|
---|
538 | while (my $line = <$file_handle>)
|
---|
539 | {
|
---|
540 | $dest_str .= $line;
|
---|
541 | }
|
---|
542 | $pass_count += &testAction(\$test_count, "comparing string written to string read", $sample_str1, $dest_str);
|
---|
543 | $pass_count += &testAction(\$test_count, "closeFileHandle() to close the readable file handle", 1, &FileUtils::closeFileHandle($path, \$file_handle));
|
---|
544 | # - cleanup
|
---|
545 | &FileUtils::removeFiles($path);
|
---|
546 | }
|
---|
547 |
|
---|
548 | sub testReadDirectory
|
---|
549 | {
|
---|
550 | my ($dir) = @_;
|
---|
551 | my $apath = $dir . '/alpha.txt';
|
---|
552 | &FileUtils::filePutContents($apath, 'alpha');
|
---|
553 | my $bpath = $dir . '/beta.txt';
|
---|
554 | &FileUtils::filePutContents($bpath, 'beta');
|
---|
555 | my $dpath = $dir . '/delta.txt';
|
---|
556 | &FileUtils::filePutContents($dpath, 'delta');
|
---|
557 | my $file_count = 3;
|
---|
558 | # - read the directory, which will only contain the files we just put there
|
---|
559 | my @files = @{&FileUtils::readDirectory($dir)};
|
---|
560 | my $result_count = 0;
|
---|
561 | foreach my $file (@files)
|
---|
562 | {
|
---|
563 | if ($file !~ /^\./)
|
---|
564 | {
|
---|
565 | $result_count++;
|
---|
566 | }
|
---|
567 | }
|
---|
568 | $pass_count += &testAction(\$test_count, "readDirectory() listing files", $file_count, $result_count);
|
---|
569 | &FileUtils::removeFiles($apath, $bpath, $dpath);
|
---|
570 | }
|
---|
571 |
|
---|
572 | ## @function testRemoveFiles()
|
---|
573 | #
|
---|
574 | sub testRemoveFiles
|
---|
575 | {
|
---|
576 | my ($dir) = @_;
|
---|
577 |
|
---|
578 | my $patha = $dir . '/alpha.txt';
|
---|
579 | &FileUtils::filePutContents($patha, 'alpha');
|
---|
580 | my $pathb = $dir . '/beta.txt';
|
---|
581 | &FileUtils::softLink($patha, $pathb);
|
---|
582 | my $pathd = $dir . '/delta.txt';
|
---|
583 | &FileUtils::filePutContents($pathd, 'delta');
|
---|
584 | my $pathe = $dir . '/epsilon.txt';
|
---|
585 | &FileUtils::filePutContents($pathe, 'epsilon');
|
---|
586 | my $pathg = $dir . '/gamma.txt';
|
---|
587 | &FileUtils::filePutContents($pathg, 'gamma');
|
---|
588 | my $patht = $dir . '/tau.txt';
|
---|
589 | my $dirm = $dir .'/mu';
|
---|
590 | &FileUtils::makeDirectory($dirm);
|
---|
591 | # - tests
|
---|
592 | $pass_count += &testAction(\$test_count, "removeFiles() for existing file", 1, &FileUtils::removeFiles($patha));
|
---|
593 | $pass_count += &testAction(\$test_count, "removeFiles() for soft symbolic link", 1, &FileUtils::removeFiles($pathb));
|
---|
594 | $pass_count += &testAction(\$test_count, "removeFiles() for multiple existing files", 3, &FileUtils::removeFiles($pathd, $pathe, $pathg));
|
---|
595 | $pass_count += &testAction(\$test_count, "removeFiles() for file that doesn't exist (should fail)", 0, &FileUtils::removeFiles($patht));
|
---|
596 | $pass_count += &testAction(\$test_count, "removeFiles() for existing directory (should fail)", 0, &FileUtils::removeFiles($dirm));
|
---|
597 | # - cleanup
|
---|
598 | &FileUtils::removeFilesRecursive($dirm);
|
---|
599 | }
|
---|
600 | ## testRemoveFiles()
|
---|
601 |
|
---|
602 |
|
---|
603 | ## @function testRemoveFilesFiltered()
|
---|
604 | #
|
---|
605 | sub testRemoveFilesFiltered
|
---|
606 | {
|
---|
607 | my ($dir) = @_;
|
---|
608 | # - setup
|
---|
609 | my $patha = $dir . '/alpha.txt';
|
---|
610 | &FileUtils::filePutContents($patha, 'alpha');
|
---|
611 | &FileUtils::filePutContents($dir . '/beta.tmp','beta');
|
---|
612 | my $pathd = $dir . '/delta.txt';
|
---|
613 | &FileUtils::filePutContents($pathd, 'delta');
|
---|
614 | &FileUtils::filePutContents($dir . '/gamma.tmp','gamma');
|
---|
615 | &FileUtils::filePutContents($dir . '/epsilon.txt','epsilon');
|
---|
616 | &FileUtils::filePutContents($dir . '/eta.txt','eta');
|
---|
617 | # - tests
|
---|
618 | $pass_count += &testAction(\$test_count, "filteredRemove() with accept regular expression", 2, &FileUtils::removeFilesFiltered($dir, '\.tmp$', undef));
|
---|
619 | $pass_count += &testAction(\$test_count, "filteredRemove() with accept and reject regular expression", 2, &FileUtils::removeFilesFiltered($dir, '\.txt$', '(alpha|delta)'));
|
---|
620 | # - cleanup
|
---|
621 | &FileUtils::removeFiles($patha, $pathd);
|
---|
622 | }
|
---|
623 | ## testRemoveFilesFiltered()
|
---|
624 |
|
---|
625 |
|
---|
626 | ## @function testRemoveFilesRecursive()
|
---|
627 | #
|
---|
628 | sub testRemoveFilesRecursive
|
---|
629 | {
|
---|
630 | my ($dir) = @_;
|
---|
631 | # - setup
|
---|
632 | my $dira = $dir . '/alpha';
|
---|
633 | &FileUtils::makeDirectory($dira);
|
---|
634 | &FileUtils::filePutContents($dira . '/alpha.txt', 'alpha');
|
---|
635 | &FileUtils::filePutContents($dira . '/beta.txt', 'beta');
|
---|
636 | my $dirb = $dira . '/beta';
|
---|
637 | &FileUtils::makeDirectory($dirb);
|
---|
638 | &FileUtils::filePutContents($dirb . '/gamma.txt', 'gamma');
|
---|
639 | # - test
|
---|
640 | $pass_count += &testAction(\$test_count, "removeFilesRecursive() for directory containing files", 5, &FileUtils::removeFilesRecursive($dira));
|
---|
641 | }
|
---|
642 | ## testRemoveFilesRecursive()
|
---|
643 |
|
---|
644 |
|
---|
645 | ## @function testTransferFiles()
|
---|
646 | #
|
---|
647 | sub testTransferFiles
|
---|
648 | {
|
---|
649 | my ($dir, $mode, $local_dir) = @_;
|
---|
650 | my $verb = 'moving';
|
---|
651 | if ($mode eq 'copy')
|
---|
652 | {
|
---|
653 | $verb = 'copying';
|
---|
654 | }
|
---|
655 | # Move a single local file to a new local location
|
---|
656 | # - setup
|
---|
657 | my $apath = $dir . '/alpha.txt';
|
---|
658 | &FileUtils::filePutContents($apath,'alpha');
|
---|
659 | my $bpath = $dir . '/beta.txt';
|
---|
660 | my $sub_pass_count = 0;
|
---|
661 | # - subtests
|
---|
662 | if ($mode eq 'move')
|
---|
663 | {
|
---|
664 | $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($apath, $bpath));
|
---|
665 | $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($apath));
|
---|
666 | }
|
---|
667 | else
|
---|
668 | {
|
---|
669 | $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($apath, $bpath));
|
---|
670 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
|
---|
671 | }
|
---|
672 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($bpath));
|
---|
673 |
|
---|
674 | # - test
|
---|
675 | $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " a single file", 3, $sub_pass_count);
|
---|
676 | # - cleanup
|
---|
677 | &FileUtils::removeFiles($bpath);
|
---|
678 |
|
---|
679 | # Move multiple local files into a single local directory
|
---|
680 | # - setup
|
---|
681 | my $dpath = $dir . '/delta.txt';
|
---|
682 | &FileUtils::filePutContents($dpath, 'delta');
|
---|
683 | my $gpath = $dir . '/gamma.txt';
|
---|
684 | &FileUtils::filePutContents($gpath, 'gamma');
|
---|
685 | my $move_dir = $dir . '/movedir';
|
---|
686 | &FileUtils::makeDirectory($move_dir);
|
---|
687 | $sub_pass_count = 0;
|
---|
688 | # - subtests
|
---|
689 | if ($mode eq 'move')
|
---|
690 | {
|
---|
691 | $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($dpath, $gpath, $move_dir));
|
---|
692 | $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($dpath));
|
---|
693 | $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($gpath));
|
---|
694 | }
|
---|
695 | else
|
---|
696 | {
|
---|
697 | $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($dpath, $gpath, $move_dir));
|
---|
698 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($dpath));
|
---|
699 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($gpath));
|
---|
700 | }
|
---|
701 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($move_dir . '/delta.txt'));
|
---|
702 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($move_dir . '/gamma.txt'));
|
---|
703 | # - test
|
---|
704 | $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a directory", 5, $sub_pass_count);
|
---|
705 | # - cleanup
|
---|
706 | if ($mode ne 'move')
|
---|
707 | {
|
---|
708 | &FileUtils::removeFiles($dpath, $gpath);
|
---|
709 | }
|
---|
710 |
|
---|
711 | # Moving multiple files to file
|
---|
712 | if ($mode eq 'move')
|
---|
713 | {
|
---|
714 | $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a file (should fail)", 0, &FileUtils::moveFiles($move_dir . '/delta.txt', $move_dir . '/gamma.txt', $bpath));
|
---|
715 | }
|
---|
716 | else
|
---|
717 | {
|
---|
718 | $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a file (should fail)", 0, &FileUtils::copyFiles($move_dir . '/delta.txt', $move_dir . '/gamma.txt', $bpath));
|
---|
719 | }
|
---|
720 |
|
---|
721 | # - cleanup
|
---|
722 | &FileUtils::removeFilesRecursive($move_dir);
|
---|
723 |
|
---|
724 | if (defined $local_dir && -d $local_dir)
|
---|
725 | {
|
---|
726 | &FileUtils::filePutContents($apath, 'alpha');
|
---|
727 | my $zpath = $local_dir . '/fileutils-' . time() . '.txt';
|
---|
728 | # - init
|
---|
729 | $sub_pass_count = 0;
|
---|
730 | # - subtests
|
---|
731 | if ($mode eq 'move')
|
---|
732 | {
|
---|
733 | $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($apath, $zpath));
|
---|
734 | $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($apath));
|
---|
735 | }
|
---|
736 | else
|
---|
737 | {
|
---|
738 | $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($apath, $zpath));
|
---|
739 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
|
---|
740 | }
|
---|
741 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($zpath));
|
---|
742 | # - test
|
---|
743 | $pass_count += &testAction(\$test_count, $mode . "Files() from non-local file to local one", 3, $sub_pass_count);
|
---|
744 |
|
---|
745 | # If we are copying, we'll need to remove alpha, otherwise the round trip
|
---|
746 | # back to HDFS will fail
|
---|
747 | if ($mode eq 'copy')
|
---|
748 | {
|
---|
749 | &FileUtils::removeFiles($apath);
|
---|
750 | }
|
---|
751 |
|
---|
752 | # - init
|
---|
753 | $sub_pass_count = 0;
|
---|
754 | # - subtests
|
---|
755 | if ($mode eq 'move')
|
---|
756 | {
|
---|
757 | $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($zpath, $apath));
|
---|
758 | $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($zpath));
|
---|
759 | }
|
---|
760 | else
|
---|
761 | {
|
---|
762 | $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($zpath, $apath));
|
---|
763 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($zpath));
|
---|
764 | }
|
---|
765 | $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
|
---|
766 |
|
---|
767 | # - test
|
---|
768 | $pass_count += &testAction(\$test_count, $mode . "Files() from local file to non-local one", 3, $sub_pass_count);
|
---|
769 | # - cleanup
|
---|
770 | if ($mode eq 'move')
|
---|
771 | {
|
---|
772 | &FileUtils::removeFiles($apath);
|
---|
773 | }
|
---|
774 | else
|
---|
775 | {
|
---|
776 | &FileUtils::removeFiles($apath, $zpath);
|
---|
777 | }
|
---|
778 | }
|
---|
779 | else
|
---|
780 | {
|
---|
781 | # Move a single non-local file to a new local location
|
---|
782 | $skip_count += &skipAction(\$test_count, $mode . "Files() from non-local file to local one");
|
---|
783 | # Move a single local file to a new non-local location
|
---|
784 | $skip_count += &skipAction(\$test_count, $mode . "Files() from local file to non-local one");
|
---|
785 | }
|
---|
786 | }
|
---|
787 | ## testTransferFiles()
|
---|
788 |
|
---|
789 | 1;
|
---|