source: gs2-extensions/parallel-building/trunk/src/perllib/util.pm@ 26984

Last change on this file since 26984 was 26984, checked in by jmt12, 11 years ago

Removing some debug comments
File size: 61.4 KB
Line 
1###########################################################################
2#
3# util.pm -- various useful utilities
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26package util;
27
28use strict;
29
30use Encode;
31use File::Copy;
32use File::Basename;
33# Config for getting the perlpath in the recommended way, though it uses paths that are
34# hard-coded into the Config file that's generated upon configuring and compiling perl.
35# $^X works better in some cases to return the path to perl used to launch the script,
36# but if launched with plain "perl" (no full-path), that will be just what it returns.
37use Config;
38
39# removes files (but not directories)
40sub rm {
41 my (@files) = @_;
42
43 my @filefiles = ();
44
45 # make sure the files we want to delete exist
46 # and are regular files
47 foreach my $file (@files) {
48
49 if (&util::isHDFS($file))
50 {
51 &util::executeHDFSCommand('rm', $file);
52 }
53 else
54 {
55 if (!-e $file) {
56 print STDERR "util::rm $file does not exist\n";
57 } elsif ((!-f $file) && (!-l $file)) {
58 print STDERR "util::rm $file is not a regular (or symbolic) file\n";
59 } else {
60 push (@filefiles, $file);
61 }
62 }
63 }
64
65 # remove the files
66 my $numremoved = unlink @filefiles;
67
68 # check to make sure all of them were removed
69 if ($numremoved != scalar(@filefiles)) {
70 print STDERR "util::rm Not all files were removed\n";
71 }
72}
73
74# removes files (but not directories) - can rename this to the default
75# "rm" subroutine when debugging the deletion of individual files.
76sub rm_debug {
77 my (@files) = @_;
78 my @filefiles = ();
79
80 # make sure the files we want to delete exist
81 # and are regular files
82 foreach my $file (@files) {
83 if (!-e $file) {
84 print STDERR "util::rm $file does not exist\n";
85 } elsif ((!-f $file) && (!-l $file)) {
86 print STDERR "util::rm $file is not a regular (or symbolic) file\n";
87 } else { # debug message
88 unlink($file) or warn "Could not delete file $file: $!\n";
89 }
90 }
91}
92
93
94# recursive removal
95sub filtered_rm_r {
96 my ($files,$file_accept_re,$file_reject_re) = @_;
97
98# my ($cpackage,$cfilename,$cline,$csubr,$chas_args,$cwantarray) = caller(2);
99# my ($lcfilename) = ($cfilename =~ m/([^\\\/]*)$/);
100# print STDERR "** Calling method (2): $lcfilename:$cline $cpackage->$csubr\n";
101
102 my @files_array = (ref $files eq "ARRAY") ? @$files : ($files);
103
104 # recursively remove the files
105 foreach my $file (@files_array) {
106
107 # HDFS support
108 if (&util::isHDFS($file))
109 {
110 # HDFS doesn't really lend itself to a choosy delete, unless you want
111 # it to be really, really, unbearably slow.
112 &util::executeHDFSCommand('rmr', $file);
113 next;
114 }
115
116
117 $file =~ s/[\/\\]+$//; # remove trailing slashes
118
119 if (!-e $file) {
120 print STDERR "util::filtered_rm_r $file does not exist\n";
121
122 } elsif ((-d $file) && (!-l $file)) { # don't recurse down symbolic link
123 # get the contents of this directory
124 if (!opendir (INDIR, $file)) {
125 print STDERR "util::filtered_rm_r could not open directory $file\n";
126 } else {
127 my @filedir = grep (!/^\.\.?$/, readdir (INDIR));
128 closedir (INDIR);
129
130 # remove all the files in this directory
131 map {$_="$file/$_";} @filedir;
132 &filtered_rm_r (\@filedir,$file_accept_re,$file_reject_re);
133
134 if (!defined $file_accept_re && !defined $file_reject_re) {
135 # remove this directory
136 if (!rmdir $file) {
137 print STDERR "util::filtered_rm_r couldn't remove directory $file\n";
138 }
139 }
140 }
141 } else {
142 next if (defined $file_reject_re && ($file =~ m/$file_reject_re/));
143
144 if ((!defined $file_accept_re) || ($file =~ m/$file_accept_re/)) {
145 # remove this file
146 &rm ($file);
147 }
148 }
149 }
150}
151
152
153# recursive removal
154sub rm_r
155{
156 my (@files) = @_;
157 # use the more general (but reterospectively written function
158 # filtered_rm_r function()
159 filtered_rm_r(\@files,undef,undef); # no accept or reject expressions
160}
161
162
163
164
165# moves a file or a group of files
166sub mv {
167 my $dest = pop (@_);
168 my (@srcfiles) = @_;
169
170 # moving a file within or into HDFS
171 if (&util::isHDFS($dest))
172 {
173 foreach my $src (@srcfiles)
174 {
175 if (&util::isHDFS($src))
176 {
177 &util::executeHDFSCommand('mv', $src, $dest);
178 }
179 else
180 {
181 &util::executeHDFSCommand('put', $src, $dest);
182 &util::rm_r($src);
183 }
184 }
185 return;
186 }
187
188 # remove trailing slashes from source and destination files
189 $dest =~ s/[\\\/]+$//;
190 map {$_ =~ s/[\\\/]+$//;} @srcfiles;
191
192 # a few sanity checks
193 if (scalar (@srcfiles) == 0) {
194 print STDERR "util::mv no destination directory given\n";
195 return;
196 } elsif ((scalar (@srcfiles) > 1) && (!-d $dest)) {
197 print STDERR "util::mv if multiple source files are given the ".
198 "destination must be a directory\n";
199 return;
200 }
201
202 # move the files
203 foreach my $file (@srcfiles) {
204
205 # moving a file out of HDFS
206 if (&util::isHDFS($file))
207 {
208 &util::executeHDFSCommand('get', $file, $dest);
209 &util::rm_r($file);
210 next;
211 }
212
213 my $tempdest = $dest;
214 if (-d $tempdest) {
215 my ($filename) = $file =~ /([^\\\/]+)$/;
216 $tempdest .= "/$filename";
217 }
218 if (!-e $file) {
219 print STDERR "util::mv $file does not exist\n";
220 } else {
221 if(!rename ($file, $tempdest)) {
222 print STDERR "**** Failed to rename $file to $tempdest\n";
223 &File::Copy::copy($file, $tempdest);
224 &rm($file);
225 }
226 elsif(-e $file) { # rename (partially) succeeded) but srcfile still exists after rename
227 #print STDERR "*** srcfile $file still exists after rename to $tempdest\n";
228 if(!-e $tempdest) {
229 print STDERR "@@@@ ERROR: $tempdest does not exist\n";
230 }
231 # Sometimes the rename operation fails (as does File::Copy::move).
232 # This turns out to be because the files are hardlinked.
233 # Need to do a copy-delete in this case, however, the copy step is not necessary:
234 # the srcfile got renamed into tempdest, but srcfile itself still exists, delete it.
235 #&File::Copy::copy($file, $tempdest);
236
237 &rm($file);
238 }
239 }
240 }
241}
242
243# Move the contents of source directory into target directory
244# (as opposed to merely replacing target dir with the src dir)
245# This can overwrite any files with duplicate names in the target
246# but other files and folders in the target will continue to exist
247sub mv_dir_contents {
248 my ($src_dir, $dest_dir) = @_;
249
250 # Obtain listing of all files within src_dir
251 # Note that readdir lists relative paths, as well as . and ..
252 opendir(DIR, "$src_dir");
253 my @files= readdir(DIR);
254 close(DIR);
255
256 my @full_path_files = ();
257 foreach my $file (@files) {
258 # process all except . and ..
259 unless($file eq "." || $file eq "..") {
260
261 my $dest_subdir = &filename_cat($dest_dir, $file); # $file is still a relative path
262
263 # construct absolute paths
264 $file = &filename_cat($src_dir, $file); # $file is now an absolute path
265
266 # Recurse on directories which have an equivalent in target dest_dir
267 # If $file is a directory that already exists in target $dest_dir,
268 # then a simple move operation will fail (definitely on Windows).
269 if(-d $file && -d $dest_subdir) {
270 #print STDERR "**** $file is a directory also existing in target, its contents to be copied to $dest_subdir\n";
271 &mv_dir_contents($file, $dest_subdir);
272
273 # now all content is moved across, delete empty dir in source folder
274 if(&is_dir_empty($file)) {
275 if (!rmdir $file) {
276 print STDERR "ERROR. util::mv_dir_contents couldn't remove directory $file\n";
277 }
278 } else { # error
279 print STDERR "ERROR. util::mv_dir_contents: subfolder $file still non-empty after moving contents to $dest_subdir\n";
280 }
281 } else { # process files and any directories that don't already exist with a simple move
282 push(@full_path_files, $file);
283 }
284 }
285 }
286
287 if(!&dir_exists($dest_dir)) { # create target toplevel folder or subfolders if they don't exist
288 &mk_dir($dest_dir);
289 }
290
291 #print STDERR "@@@@@ Copying files |".join(",", @full_path_files)."| to: $dest_dir\n";
292
293 if(@full_path_files) { # if non-empty, there's something to copy across
294 &mv(@full_path_files, $dest_dir);
295 }
296}
297
298
299# copies a file or a group of files
300sub cp
301{
302 my $dest = pop (@_);
303 my (@srcfiles) = @_;
304
305 # remove trailing slashes from source and destination files
306 $dest =~ s/[\\\/]+$//;
307 map {$_ =~ s/[\\\/]+$//;} @srcfiles;
308
309 # a few sanity checks
310 if (scalar (@srcfiles) == 0)
311 {
312 print STDERR "util::cp no destination directory given\n";
313 return 0;
314 }
315 elsif ((scalar (@srcfiles) > 1) && (!&util::dir_exists($dest)))
316 {
317 print STDERR "util::cp if multiple source files are given the destination must be a directory\n";
318 return 0;
319 }
320
321 # copying a file into or within HDFS
322 if (&util::isHDFS($dest))
323 {
324 foreach my $src (@srcfiles)
325 {
326 &util::executeHDFSCommand('put', $src, $dest);
327 &util::rm_r($src);
328 }
329 return;
330 }
331
332 # copy the files
333 foreach my $file (@srcfiles)
334 {
335 my $tempdest = $dest;
336 if (&util::dir_exists($tempdest))
337 {
338 my ($filename) = $file =~ /([^\\\/]+)$/;
339 $tempdest .= "/$filename";
340 }
341 if (!&util::file_exists($file))
342 {
343 if (&util::dir_exists($file))
344 {
345 print STDERR "util::cp $file is not a plain file\n";
346 }
347 else
348 {
349 print STDERR "util::cp $file does not exist\n";
350 }
351 }
352 elsif (&util::isHDFS($file))
353 {
354 &util::executeHDFSCommand('get', $file, $dest);
355 }
356 else
357 {
358 &File::Copy::copy ($file, $tempdest);
359 }
360 }
361}
362
363
364
365# recursively copies a file or group of files
366# syntax: cp_r (sourcefiles, destination directory)
367# destination must be a directory - to copy one file to
368# another use cp instead
369sub cp_r {
370 my $dest = pop (@_);
371 my (@srcfiles) = @_;
372
373 # a few sanity checks
374 if (scalar (@srcfiles) == 0) {
375 print STDERR "util::cp_r no destination directory given\n";
376 return;
377 } elsif (-f $dest) {
378 print STDERR "util::cp_r destination must be a directory\n";
379 return;
380 }
381
382 # create destination directory if it doesn't exist already
383 if (! -d $dest) {
384 my $store_umask = umask(0002);
385 mkdir ($dest, 0777);
386 umask($store_umask);
387 }
388
389 # copy the files
390 foreach my $file (@srcfiles) {
391
392 if (!-e $file) {
393 print STDERR "util::cp_r $file does not exist\n";
394
395 } elsif (-d $file) {
396 # make the new directory
397 my ($filename) = $file =~ /([^\\\/]*)$/;
398 $dest = &util::filename_cat ($dest, $filename);
399 my $store_umask = umask(0002);
400 mkdir ($dest, 0777);
401 umask($store_umask);
402
403 # get the contents of this directory
404 if (!opendir (INDIR, $file)) {
405 print STDERR "util::cp_r could not open directory $file\n";
406 } else {
407 my @filedir = readdir (INDIR);
408 closedir (INDIR);
409 foreach my $f (@filedir) {
410 next if $f =~ /^\.\.?$/;
411 # copy all the files in this directory
412 my $ff = &util::filename_cat ($file, $f);
413 &cp_r ($ff, $dest);
414 }
415 }
416
417 } else {
418 &cp($file, $dest);
419 }
420 }
421}
422# recursively copies a file or group of files
423# syntax: cp_r (sourcefiles, destination directory)
424# destination must be a directory - to copy one file to
425# another use cp instead
426sub cp_r_nosvn {
427 my $dest = pop (@_);
428 my (@srcfiles) = @_;
429
430 # a few sanity checks
431 if (scalar (@srcfiles) == 0) {
432 print STDERR "util::cp_r no destination directory given\n";
433 return;
434 } elsif (-f $dest) {
435 print STDERR "util::cp_r destination must be a directory\n";
436 return;
437 }
438
439 # create destination directory if it doesn't exist already
440 if (! -d $dest) {
441 my $store_umask = umask(0002);
442 mkdir ($dest, 0777);
443 umask($store_umask);
444 }
445
446 # copy the files
447 foreach my $file (@srcfiles) {
448
449 if (!-e $file) {
450 print STDERR "util::cp_r $file does not exist\n";
451
452 } elsif (-d $file) {
453 # make the new directory
454 my ($filename) = $file =~ /([^\\\/]*)$/;
455 $dest = &util::filename_cat ($dest, $filename);
456 my $store_umask = umask(0002);
457 mkdir ($dest, 0777);
458 umask($store_umask);
459
460 # get the contents of this directory
461 if (!opendir (INDIR, $file)) {
462 print STDERR "util::cp_r could not open directory $file\n";
463 } else {
464 my @filedir = readdir (INDIR);
465 closedir (INDIR);
466 foreach my $f (@filedir) {
467 next if $f =~ /^\.\.?$/;
468 next if $f =~ /^\.svn$/;
469 # copy all the files in this directory
470 my $ff = &util::filename_cat ($file, $f);
471 &cp_r ($ff, $dest);
472 }
473 }
474
475 } else {
476 &cp($file, $dest);
477 }
478 }
479}
480
481# copies a directory and its contents, excluding subdirectories, into a new directory
482sub cp_r_toplevel {
483 my $dest = pop (@_);
484 my (@srcfiles) = @_;
485
486 # a few sanity checks
487 if (scalar (@srcfiles) == 0) {
488 print STDERR "util::cp_r no destination directory given\n";
489 return;
490 } elsif (-f $dest) {
491 print STDERR "util::cp_r destination must be a directory\n";
492 return;
493 }
494
495 # create destination directory if it doesn't exist already
496 if (! -d $dest) {
497 my $store_umask = umask(0002);
498 mkdir ($dest, 0777);
499 umask($store_umask);
500 }
501
502 # copy the files
503 foreach my $file (@srcfiles) {
504
505 if (!-e $file) {
506 print STDERR "util::cp_r $file does not exist\n";
507
508 } elsif (-d $file) {
509 # make the new directory
510 my ($filename) = $file =~ /([^\\\/]*)$/;
511 $dest = &util::filename_cat ($dest, $filename);
512 my $store_umask = umask(0002);
513 mkdir ($dest, 0777);
514 umask($store_umask);
515
516 # get the contents of this directory
517 if (!opendir (INDIR, $file)) {
518 print STDERR "util::cp_r could not open directory $file\n";
519 } else {
520 my @filedir = readdir (INDIR);
521 closedir (INDIR);
522 foreach my $f (@filedir) {
523 next if $f =~ /^\.\.?$/;
524
525 # copy all the files in this directory, but not directories
526 my $ff = &util::filename_cat ($file, $f);
527 if (-f $ff) {
528 &cp($ff, $dest);
529 #&cp_r ($ff, $dest);
530 }
531 }
532 }
533
534 } else {
535 &cp($file, $dest);
536 }
537 }
538}
539
540# /** @function mk_dir()
541# * Extend mkdir to allow it to silently fail in the case where code is
542# * 'competing' to create a directory first.
543# * @param $dir the full path of the directory to create
544# * @param $can_fail 1 if the mkdir can fail silently (optional)
545# * @return 1 on success, 0 on failure
546# */
547sub mk_dir
548{
549 my ($dir, $can_fail) = @_;
550 my $mkdir_ok = 0;
551 if (&util::isHDFS($dir))
552 {
553 # unhelpfully HDFS mkdir returns 0 on success, -1 on failure
554 my $result = &util::executeHDFSCommand('mkdir', $dir);
555 if ($result == 0)
556 {
557 $mkdir_ok = 1;
558 }
559 }
560 else
561 {
562 my $store_umask = umask(0002);
563 my $mkdir_ok = mkdir ($dir, 0777);
564 umask($store_umask);
565 }
566 # only output an error if this call wasn't marked as can_fail
567 if (!$mkdir_ok && (!defined $can_fail || !$can_fail))
568 {
569 print STDERR "util::mk_dir could not create directory: $dir\n error: $!\n";
570 }
571 return $mkdir_ok;
572}
573
574# in case anyone cares - I did some testing (using perls Benchmark module)
575# on this subroutine against File::Path::mkpath (). mk_all_dir() is apparently
576# slightly faster (surprisingly) - Stefan.
577sub mk_all_dir
578{
579 my ($dir) = @_;
580
581 ###rint "-> util::mk_all_dir($dir)\n";
582
583 # support for HDFS
584 if (&util::isHDFS($dir))
585 {
586 # HDFS's version of mkdir does it recursively anyway
587 my $result = &util::executeHDFSCommand('mkdir', $dir);
588 return ($result == 0);
589 }
590
591 # use / for the directory separator, remove duplicate and
592 # trailing slashes
593 $dir=~s/[\\\/]+/\//g;
594 $dir=~s/[\\\/]+$//;
595
596 # ensure the directory doesn't already exist
597 if (-e $dir)
598 {
599 return 0;
600 }
601
602 # make sure the cache directory exists
603 my $dirsofar = "";
604 my $first = 1;
605 foreach my $dirname (split ("/", $dir))
606 {
607 $dirsofar .= "/" unless $first;
608 $first = 0;
609
610 $dirsofar .= $dirname;
611
612 next if $dirname =~ /^(|[a-z]:)$/i;
613 if (!-e $dirsofar)
614 {
615 my $store_umask = umask(0002);
616 my $mkdir_ok = mkdir ($dirsofar, 0777);
617 umask($store_umask);
618 if (!$mkdir_ok)
619 {
620 print STDERR "util::mk_all_dir could not create directory $dirsofar\n";
621 return 0;
622 }
623 }
624 }
625 return (-e $dir);
626}
627
628# make hard link to file if supported by OS, otherwise copy the file
629sub hard_link {
630 my ($src, $dest, $verbosity) = @_;
631
632 # remove trailing slashes from source and destination files
633 $src =~ s/[\\\/]+$//;
634 $dest =~ s/[\\\/]+$//;
635
636## print STDERR "**** src = ", unicode::debug_unicode_string($src),"\n";
637 # a few sanity checks
638 if (&util::file_exists($dest)) {
639 # destination file already exists
640 return;
641 }
642 elsif (!&util::file_exists($src)) {
643 print STDERR "util::hard_link source file \"$src\" does not exist\n";
644 return 1;
645 }
646 elsif (!&util::file_exists($src) && &util::dir_exists($src)) {
647 print STDERR "util::hard_link source \"$src\" is a directory\n";
648 return 1;
649 }
650
651 my $dest_dir = &File::Basename::dirname($dest);
652 if (!&util::dir_exists($dest_dir))
653 {
654 mk_all_dir($dest_dir);
655 }
656
657 # HDFS Support - we can't ever link, copy instead
658 if (&util::isHDFS($src))
659 {
660 if (&util::isHDFS($dest))
661 {
662 &util::executeHDFSCommand('put', $src, $dest);
663 return 0;
664 }
665 else
666 {
667 &util::executeHDFSCommand('get', $src, $dest);
668 return 0;
669 }
670 }
671 elsif (&util::isHDFS($dest))
672 {
673 &util::executeHDFSCommand('put', $src, $dest);
674 return 0;
675 }
676
677 if (!link($src, $dest)) {
678 if ((!defined $verbosity) || ($verbosity>2)) {
679 print STDERR "util::hard_link: unable to create hard link. ";
680 print STDERR " Copying file: $src -> $dest\n";
681 }
682 &File::Copy::copy ($src, $dest);
683 }
684 return 0;
685}
686
687# make soft link to file if supported by OS, otherwise copy file
688sub soft_link {
689 my ($src, $dest, $ensure_paths_absolute) = @_;
690
691 # remove trailing slashes from source and destination files
692 $src =~ s/[\\\/]+$//;
693 $dest =~ s/[\\\/]+$//;
694
695 # Ensure file paths are absolute IF requested to do so
696 # Soft_linking didn't work for relative paths
697 if(defined $ensure_paths_absolute && $ensure_paths_absolute) {
698 # We need to ensure that the src file is the absolute path
699 # See http://perldoc.perl.org/File/Spec.html
700 if(!File::Spec->file_name_is_absolute( $src )) { # it's relative
701 $src = File::Spec->rel2abs($src); # make absolute
702 }
703 # Might as well ensure that the destination file's absolute path is used
704 if(!File::Spec->file_name_is_absolute( $dest )) {
705 $dest = File::Spec->rel2abs($dest); # make absolute
706 }
707 }
708
709 # a few sanity checks
710 if (!-e $src) {
711 print STDERR "util::soft_link source file $src does not exist\n";
712 return 0;
713 }
714
715 my $dest_dir = &File::Basename::dirname($dest);
716 mk_all_dir($dest_dir) if (!-e $dest_dir);
717
718 if ($ENV{'GSDLOS'} =~ /^windows$/i) {
719
720 # symlink not supported on windows
721 &File::Copy::copy ($src, $dest);
722
723 } elsif (!eval {symlink($src, $dest)}) {
724 print STDERR "util::soft_link: unable to create soft link.\n";
725 return 0;
726 }
727
728 return 1;
729}
730
731# Primarily for filenames generated by processing
732# content of HTML files (which are mapped to UTF-8 internally)
733#
734# To turn this into an octet string that really exists on the file
735# system:
736# 1. don't need to do anything special for Unix-based systems
737# (as underlying file system is byte-code)
738# 2. need to map to short DOS filenames for Windows
739
740sub utf8_to_real_filename
741{
742 my ($utf8_filename) = @_;
743
744 my $real_filename;
745
746 if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
747 require Win32;
748
749 print STDERR "***** utf8 filename = $utf8_filename\n\n\n";
750
751 my $unicode_filename = decode("utf8",$utf8_filename);
752 $real_filename = Win32::GetShortPathName($unicode_filename);
753 }
754 else {
755 $real_filename = $utf8_filename;
756 }
757
758 return $real_filename;
759}
760
761sub fd_exists
762{
763 my $filename_full_path = shift @_;
764 my $test_op = shift @_ || "-e";
765
766 # By default tests for existance of file or directory (-e)
767 # Can be made more specific by providing second parameter (e.g. -f or -d)
768
769 my $exists = 0;
770
771 # Support for HDFS [jmt12]
772 if (&util::isHDFS($filename_full_path))
773 {
774 # very limited test op support for HDFS
775 if ($test_op ne '-d' && $test_op ne '-e' && $test_op ne '-z')
776 {
777 $test_op = '-e';
778 }
779 my $result = &util::executeHDFSCommand('test ' . $test_op, $filename_full_path);
780 return ($result == 0);
781 }
782
783 if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
784 require Win32;
785 my $filename_short_path = Win32::GetShortPathName($filename_full_path);
786 if (!defined $filename_short_path) {
787 # Was probably still in UTF8 form (not what is needed on Windows)
788 my $unicode_filename_full_path = eval "decode(\"utf8\",\$filename_full_path)";
789 if (defined $unicode_filename_full_path) {
790 $filename_short_path = Win32::GetShortPathName($unicode_filename_full_path);
791 }
792 }
793 $filename_full_path = $filename_short_path;
794 }
795
796 if (defined $filename_full_path) {
797 $exists = eval "($test_op \$filename_full_path)";
798 }
799
800 return $exists;
801}
802
803sub file_exists
804{
805 my ($filename_full_path) = @_;
806# if ($filename_full_path =~ /^hdfs:/)
807# {
808# print "-> util::file_exists(" . $filename_full_path . ")\n";
809# }
810 return fd_exists($filename_full_path,"-f");
811}
812
813sub dir_exists
814{
815 my ($filename_full_path) = @_;
816# if ($filename_full_path =~ /^hdfs:/)
817# {
818# print "-> util::dir_exists(" . $filename_full_path . ")\n";
819# }
820 return fd_exists($filename_full_path,"-d");
821}
822
823
824
825# updates a copy of a directory in some other part of the filesystem
826# verbosity settings are: 0=low, 1=normal, 2=high
827# both $fromdir and $todir should be absolute paths
828sub cachedir {
829 my ($fromdir, $todir, $verbosity) = @_;
830 $verbosity = 1 unless defined $verbosity;
831
832 # use / for the directory separator, remove duplicate and
833 # trailing slashes
834 $fromdir=~s/[\\\/]+/\//g;
835 $fromdir=~s/[\\\/]+$//;
836 $todir=~s/[\\\/]+/\//g;
837 $todir=~s/[\\\/]+$//;
838
839 &mk_all_dir ($todir);
840
841 # get the directories in ascending order
842 if (!opendir (FROMDIR, $fromdir)) {
843 print STDERR "util::cachedir could not read directory $fromdir\n";
844 return;
845 }
846 my @fromdir = grep (!/^\.\.?$/, sort(readdir (FROMDIR)));
847 closedir (FROMDIR);
848
849 if (!opendir (TODIR, $todir)) {
850 print STDERR "util::cacedir could not read directory $todir\n";
851 return;
852 }
853 my @todir = grep (!/^\.\.?$/, sort(readdir (TODIR)));
854 closedir (TODIR);
855
856 my $fromi = 0;
857 my $toi = 0;
858
859 while ($fromi < scalar(@fromdir) || $toi < scalar(@todir)) {
860# print "fromi: $fromi toi: $toi\n";
861
862 # see if we should delete a file/directory
863 # this should happen if the file/directory
864 # is not in the from list or if its a different
865 # size, or has an older timestamp
866 if ($toi < scalar(@todir)) {
867 if (($fromi >= scalar(@fromdir)) ||
868 ($todir[$toi] lt $fromdir[$fromi] ||
869 ($todir[$toi] eq $fromdir[$fromi] &&
870 &differentfiles("$fromdir/$fromdir[$fromi]","$todir/$todir[$toi]",
871 $verbosity)))) {
872
873 # the files are different
874 &rm_r("$todir/$todir[$toi]");
875 splice(@todir, $toi, 1); # $toi stays the same
876
877 } elsif ($todir[$toi] eq $fromdir[$fromi]) {
878 # the files are the same
879 # if it is a directory, check its contents
880 if (-d "$todir/$todir[$toi]") {
881 &cachedir ("$fromdir/$fromdir[$fromi]",
882 "$todir/$todir[$toi]", $verbosity);
883 }
884
885 $toi++;
886 $fromi++;
887 next;
888 }
889 }
890
891 # see if we should insert a file/directory
892 # we should insert a file/directory if there
893 # is no tofiles left or if the tofile does not exist
894 if ($fromi < scalar(@fromdir) && ($toi >= scalar(@todir) ||
895 $todir[$toi] gt $fromdir[$fromi])) {
896 &cp_r ("$fromdir/$fromdir[$fromi]", "$todir/$fromdir[$fromi]");
897 splice (@todir, $toi, 0, $fromdir[$fromi]);
898
899 $toi++;
900 $fromi++;
901 }
902 }
903}
904
905# this function returns -1 if either file is not found
906# assumes that $file1 and $file2 are absolute file names or
907# in the current directory
908# $file2 is allowed to be newer than $file1
909sub differentfiles {
910 my ($file1, $file2, $verbosity) = @_;
911 $verbosity = 1 unless defined $verbosity;
912
913 $file1 =~ s/\/+$//;
914 $file2 =~ s/\/+$//;
915
916 my ($file1name) = $file1 =~ /\/([^\/]*)$/;
917 my ($file2name) = $file2 =~ /\/([^\/]*)$/;
918
919 return -1 unless (-e $file1 && -e $file2);
920 if ($file1name ne $file2name) {
921 print STDERR "filenames are not the same\n" if ($verbosity >= 2);
922 return 1;
923 }
924
925 my @file1stat = stat ($file1);
926 my @file2stat = stat ($file2);
927
928 if (-d $file1) {
929 if (! -d $file2) {
930 print STDERR "one file is a directory\n" if ($verbosity >= 2);
931 return 1;
932 }
933 return 0;
934 }
935
936 # both must be regular files
937 unless (-f $file1 && -f $file2) {
938 print STDERR "one file is not a regular file\n" if ($verbosity >= 2);
939 return 1;
940 }
941
942 # the size of the files must be the same
943 if ($file1stat[7] != $file2stat[7]) {
944 print STDERR "different sized files\n" if ($verbosity >= 2);
945 return 1;
946 }
947
948 # the second file cannot be older than the first
949 if ($file1stat[9] > $file2stat[9]) {
950 print STDERR "file is older\n" if ($verbosity >= 2);
951 return 1;
952 }
953
954 return 0;
955}
956
957
958sub get_tmp_filename
959{
960 my $file_ext = shift(@_) || undef;
961
962 my $opt_dot_file_ext = "";
963 if (defined $file_ext) {
964 if ($file_ext !~ m/\./) {
965 # no dot, so needs one added in at start
966 $opt_dot_file_ext = ".$file_ext"
967 }
968 else {
969 # allow for "extensions" such as _metadata.txt to be handled
970 # gracefully
971 $opt_dot_file_ext = $file_ext;
972 }
973 }
974
975 my $tmpdir = filename_cat($ENV{'GSDLHOME'}, "tmp");
976 &mk_all_dir ($tmpdir) unless -e $tmpdir;
977
978 my $count = 1000;
979 my $rand = int(rand $count);
980 my $full_tmp_filename = &filename_cat($tmpdir, "F$rand$opt_dot_file_ext");
981
982 while (-e $full_tmp_filename) {
983 $rand = int(rand $count);
984 $full_tmp_filename = &filename_cat($tmpdir, "F$rand$opt_dot_file_ext");
985 $count++;
986 }
987
988 return $full_tmp_filename;
989}
990
991sub get_timestamped_tmp_folder
992{
993
994 my $tmp_dirname;
995 if(defined $ENV{'GSDLCOLLECTDIR'}) {
996 $tmp_dirname = $ENV{'GSDLCOLLECTDIR'};
997 } elsif(defined $ENV{'GSDLHOME'}) {
998 $tmp_dirname = $ENV{'GSDLHOME'};
999 } else {
1000 return undef;
1001 }
1002
1003 $tmp_dirname = &util::filename_cat($tmp_dirname, "tmp");
1004 &util::mk_dir($tmp_dirname) if (!-e $tmp_dirname);
1005
1006 # add the timestamp into the path otherwise we can run into problems
1007 # if documents have the same name
1008 my $timestamp = time;
1009 my $time_tmp_dirname = &util::filename_cat($tmp_dirname, $timestamp);
1010 $tmp_dirname = $time_tmp_dirname;
1011 my $i = 1;
1012 while (-e $tmp_dirname) {
1013 $tmp_dirname = "$time_tmp_dirname$i";
1014 $i++;
1015 }
1016 &util::mk_dir($tmp_dirname);
1017
1018 return $tmp_dirname;
1019}
1020
1021sub get_timestamped_tmp_filename_in_collection
1022{
1023
1024 my ($input_filename, $output_ext) = @_;
1025 # derive tmp filename from input filename
1026 my ($tailname, $dirname, $suffix)
1027 = &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
1028
1029 # softlink to collection tmp dir
1030 my $tmp_dirname = &util::get_timestamped_tmp_folder();
1031 $tmp_dirname = $dirname unless defined $tmp_dirname;
1032
1033 # following two steps copied from ConvertBinaryFile
1034 # do we need them?? can't use them as is, as they use plugin methods.
1035
1036 #$tailname = $self->SUPER::filepath_to_utf8($tailname) unless &unicode::check_is_utf8($tailname);
1037
1038 # URLEncode this since htmls with images where the html filename is utf8 don't seem
1039 # to work on Windows (IE or Firefox), as browsers are looking for filesystem-encoded
1040 # files on the filesystem.
1041 #$tailname = &util::rename_file($tailname, $self->{'file_rename_method'}, "without_suffix");
1042 if (defined $output_ext) {
1043 $output_ext = ".$output_ext"; # add the dot
1044 } else {
1045 $output_ext = $suffix;
1046 }
1047 $output_ext= lc($output_ext);
1048 my $tmp_filename = &util::filename_cat($tmp_dirname, "$tailname$output_ext");
1049
1050 return $tmp_filename;
1051}
1052
1053sub get_toplevel_tmp_dir
1054{
1055 return filename_cat($ENV{'GSDLHOME'}, "tmp");
1056}
1057
1058
1059sub filename_to_regex {
1060 my $filename = shift (@_);
1061
1062 # need to make single backslashes double so that regex works
1063 $filename =~ s/\\/\\\\/g; # if ($ENV{'GSDLOS'} =~ /^windows$/i);
1064
1065 # note that the first part of a substitution is a regex, so RE chars need to be escaped,
1066 # the second part of a substitution is not a regex, so for e.g. full-stop can be specified literally
1067 $filename =~ s/\./\\./g; # in case there are extensions/other full stops, escape them
1068 $filename =~ s@\(@\\(@g; # escape brackets
1069 $filename =~ s@\)@\\)@g; # escape brackets
1070 $filename =~ s@\[@\\[@g; # escape brackets
1071 $filename =~ s@\]@\\]@g; # escape brackets
1072
1073 return $filename;
1074}
1075
1076sub unregex_filename {
1077 my $filename = shift (@_);
1078
1079 # need to put doubled backslashes for regex back to single
1080 $filename =~ s/\\\./\./g; # remove RE syntax for .
1081 $filename =~ s@\\\(@(@g; # remove RE syntax for ( => "\(" turns into "("
1082 $filename =~ s@\\\)@)@g; # remove RE syntax for ) => "\)" turns into ")"
1083 $filename =~ s@\\\[@[@g; # remove RE syntax for [ => "\[" turns into "["
1084 $filename =~ s@\\\]@]@g; # remove RE syntax for ] => "\]" turns into "]"
1085
1086 # \\ goes to \
1087 # This is the last step in reverse mirroring the order of steps in filename_to_regex()
1088 $filename =~ s/\\\\/\\/g; # remove RE syntax for \
1089 return $filename;
1090}
1091
1092sub filename_cat {
1093 my $first_file = shift(@_);
1094 my (@filenames) = @_;
1095
1096# Useful for debugging
1097# -- might make sense to call caller(0) rather than (1)??
1098# my ($cpackage,$cfilename,$cline,$csubr,$chas_args,$cwantarray) = caller(1);
1099# print STDERR "Calling method: $cfilename:$cline $cpackage->$csubr\n";
1100
1101 # If first_file is not null or empty, then add it back into the list
1102 if (defined $first_file && $first_file =~ /\S/) {
1103 unshift(@filenames, $first_file);
1104 }
1105
1106 my $filename = join("/", @filenames);
1107
1108 # remove duplicate slashes and remove the last slash
1109 if ($ENV{'GSDLOS'} =~ /^windows$/i) {
1110 $filename =~ s/[\\\/]+/\\/g;
1111 } else {
1112 $filename =~ s/[\/]+/\//g;
1113 # DB: want a filename abc\de.html to remain like this
1114 }
1115 $filename =~ s/[\\\/]$//;
1116
1117 # restore protocols if present [jmt12]
1118 $filename =~ s/(file|hdfs|https?):\/([^\/])/$1:\/\/$2/g;
1119
1120 return $filename;
1121}
1122
1123
1124sub pathname_cat {
1125 my $first_path = shift(@_);
1126 my (@pathnames) = @_;
1127
1128 # If first_path is not null or empty, then add it back into the list
1129 if (defined $first_path && $first_path =~ /\S/) {
1130 unshift(@pathnames, $first_path);
1131 }
1132
1133 my $join_char;
1134 if ($ENV{'GSDLOS'} =~ /^windows$/i) {
1135 $join_char = ";";
1136 } else {
1137 $join_char = ":";
1138 }
1139
1140 my $pathname = join($join_char, @pathnames);
1141
1142 # remove duplicate slashes
1143 if ($ENV{'GSDLOS'} =~ /^windows$/i) {
1144 $pathname =~ s/[\\\/]+/\\/g;
1145 } else {
1146 $pathname =~ s/[\/]+/\//g;
1147 # DB: want a pathname abc\de.html to remain like this
1148 }
1149
1150 return $pathname;
1151}
1152
1153
1154sub tidy_up_oid {
1155 my ($OID) = @_;
1156 if ($OID =~ /\./) {
1157 print STDERR "Warning, identifier $OID contains periods (.), removing them\n";
1158 $OID =~ s/\.//g; #remove any periods
1159 }
1160 if ($OID =~ /^\s.*\s$/) {
1161 print STDERR "Warning, identifier $OID starts or ends with whitespace. Removing it\n";
1162 # remove starting and trailing whitespace
1163 $OID =~ s/^\s+//;
1164 $OID =~ s/\s+$//;
1165 }
1166 if ($OID =~ /^[\d]*$/) {
1167 print STDERR "Warning, identifier $OID contains only digits. Prepending 'D'.\n";
1168 $OID = "D" . $OID;
1169 }
1170
1171 return $OID;
1172}
1173sub envvar_prepend {
1174 my ($var,$val) = @_;
1175
1176 # do not prepend any value/path that's already in the environment variable
1177
1178 my $escaped_val = &filename_to_regex($val); # escape any backslashes and brackets for upcoming regex
1179 if (!defined($ENV{$var})) {
1180 $ENV{$var} = "$val";
1181 }
1182 elsif($ENV{$var} !~ m/$escaped_val/) {
1183 $ENV{$var} = "$val;".$ENV{$var};
1184 }
1185}
1186
1187sub envvar_append {
1188 my ($var,$val) = @_;
1189
1190 # do not append any value/path that's already in the environment variable
1191
1192 my $escaped_val = &filename_to_regex($val); # escape any backslashes and brackets for upcoming regex
1193 if (!defined($ENV{$var})) {
1194 $ENV{$var} = "$val";
1195 }
1196 elsif($ENV{$var} !~ m/$escaped_val/) {
1197 $ENV{$var} .= ";$val";
1198 }
1199}
1200
1201
1202# splits a filename into a prefix and a tail extension using the tail_re, or
1203# if that fails, splits on the file_extension . (dot)
1204sub get_prefix_and_tail_by_regex {
1205
1206 my ($filename,$tail_re) = @_;
1207
1208 my ($file_prefix,$file_ext) = ($filename =~ m/^(.*?)($tail_re)$/);
1209 if ((!defined $file_prefix) || (!defined $file_ext)) {
1210 ($file_prefix,$file_ext) = ($filename =~ m/^(.*)(\..*?)$/);
1211 }
1212
1213 return ($file_prefix,$file_ext);
1214}
1215
1216# get full path and file only path from a base_dir (which may be empty) and
1217# file (which may contain directories)
1218sub get_full_filenames {
1219 my ($base_dir, $file) = @_;
1220
1221 my $filename_full_path = $file;
1222 # add on directory if present
1223 $filename_full_path = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\S/;
1224
1225 my $filename_no_path = $file;
1226
1227 # remove directory if present
1228 $filename_no_path =~ s/^.*[\/\\]//;
1229 return ($filename_full_path, $filename_no_path);
1230}
1231
1232# returns the path of a file without the filename -- ie. the directory the file is in
1233sub filename_head {
1234 my $filename = shift(@_);
1235
1236 if ($ENV{'GSDLOS'} =~ /^windows$/i) {
1237 $filename =~ s/[^\\\\]*$//;
1238 }
1239 else {
1240 $filename =~ s/[^\\\/]*$//;
1241 }
1242
1243 return $filename;
1244}
1245
1246
1247
1248# returns 1 if filename1 and filename2 point to the same
1249# file or directory
1250sub filenames_equal {
1251 my ($filename1, $filename2) = @_;
1252
1253 # use filename_cat to clean up trailing slashes and
1254 # multiple slashes
1255 $filename1 = filename_cat ($filename1);
1256 $filename2 = filename_cat ($filename2);
1257
1258 # filenames not case sensitive on windows
1259 if ($ENV{'GSDLOS'} =~ /^windows$/i) {
1260 $filename1 =~ tr/[A-Z]/[a-z]/;
1261 $filename2 =~ tr/[A-Z]/[a-z]/;
1262 }
1263 return 1 if $filename1 eq $filename2;
1264 return 0;
1265}
1266
1267# If filename is relative to within_dir, returns the relative path of filename to that directory
1268# with slashes in the filename returned as they were in the original (absolute) filename.
1269sub filename_within_directory
1270{
1271 my ($filename,$within_dir) = @_;
1272
1273 if ($within_dir !~ m/[\/\\]$/) {
1274 my $dirsep = &util::get_dirsep();
1275 $within_dir .= $dirsep;
1276 }
1277
1278 $within_dir = &filename_to_regex($within_dir); # escape DOS style file separator and brackets
1279 if ($filename =~ m/^$within_dir(.*)$/) {
1280 $filename = $1;
1281 }
1282
1283 return $filename;
1284}
1285
1286# If filename is relative to within_dir, returns the relative path of filename to that directory in URL format.
1287# Filename and within_dir can be any type of slashes, but will be compared as URLs (i.e. unix-style slashes).
1288# The subpath returned will also be a URL type filename.
1289sub filename_within_directory_url_format
1290{
1291 my ($filename,$within_dir) = @_;
1292
1293 # convert parameters only to / slashes if Windows
1294
1295 my $filename_urlformat = &filepath_to_url_format($filename);
1296 my $within_dir_urlformat = &filepath_to_url_format($within_dir);
1297
1298 #if ($within_dir_urlformat !~ m/\/$/) {
1299 # make sure directory ends with a slash
1300 #$within_dir_urlformat .= "/";
1301 #}
1302
1303 my $within_dir_urlformat_re = &filename_to_regex($within_dir_urlformat); # escape any special RE characters, such as brackets
1304
1305 #print STDERR "@@@@@ $filename_urlformat =~ $within_dir_urlformat_re\n";
1306
1307 # dir prefix may or may not end with a slash (this is discarded when extracting the sub-filepath)
1308 if ($filename_urlformat =~ m/^$within_dir_urlformat_re(?:\/)*(.*)$/) {
1309 $filename_urlformat = $1;
1310 }
1311
1312 return $filename_urlformat;
1313}
1314
1315# Convert parameter to use / slashes if Windows (if on Linux leave any \ as is,
1316# since on Linux it doesn't represent a file separator but an escape char).
1317sub filepath_to_url_format
1318{
1319 my ($filepath) = @_;
1320 if ($ENV{'GSDLOS'} =~ /^windows$/i) {
1321 # Only need to worry about Windows, as Unix style directories already in url-format
1322 # Convert Windows style \ => /
1323 $filepath =~ s@\\@/@g;
1324 }
1325 return $filepath;
1326}
1327
1328# regex filepaths on windows may include \\ as path separator. Convert \\ to /
1329sub filepath_regex_to_url_format
1330{
1331 my ($filepath) = @_;
1332 if ($ENV{'GSDLOS'} =~ /^windows$/i) {
1333 # Only need to worry about Windows, as Unix style directories already in url-format
1334 # Convert Windows style \\ => /
1335 $filepath =~ s@\\\\@/@g;
1336 }
1337 return $filepath;
1338
1339}
1340
1341# Like File::Basename::fileparse, but expects filepath in url format (ie only / slash for dirsep)
1342# and ignores trailing /
1343# returns (file, dirs) dirs will be empty if no subdirs
1344sub url_fileparse
1345{
1346 my ($filepath) = @_;
1347 # remove trailing /
1348 $filepath =~ s@/$@@;
1349 if ($filepath !~ m@/@) {
1350 return ($filepath, "");
1351 }
1352 my ($dirs, $file) = $filepath =~ m@(.+/)([^/]+)@;
1353 return ($file, $dirs);
1354
1355}
1356
1357
1358sub filename_within_collection
1359{
1360 my ($filename) = @_;
1361
1362 my $collect_dir = $ENV{'GSDLCOLLECTDIR'};
1363
1364 if (defined $collect_dir) {
1365
1366 # if from within GSDLCOLLECTDIR, then remove directory prefix
1367 # so source_filename is realative to it. This is done to aid
1368 # portability, i.e. the collection can be moved to somewhere
1369 # else on the file system and the archives directory will still
1370 # work. This is needed, for example in the applet version of
1371 # GLI where GSDLHOME/collect on the server will be different to
1372 # the collect directory of the remove user. Of course,
1373 # GSDLCOLLECTDIR subsequently needs to be put back on to turn
1374 # it back into a full pathname.
1375
1376 $filename = filename_within_directory($filename,$collect_dir);
1377 }
1378
1379 return $filename;
1380}
1381
1382sub prettyprint_file
1383{
1384 my ($base_dir,$file,$gli) = @_;
1385
1386 my $filename_full_path = &util::filename_cat($base_dir,$file);
1387
1388 if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
1389 require Win32;
1390
1391 # For some reason base_dir in the form c:/a/b/c
1392 # This leads to confusion later on, so turn it back into
1393 # the more usual Windows form
1394 $base_dir =~ s/\//\\/g;
1395 my $long_base_dir = Win32::GetLongPathName($base_dir);
1396 my $long_full_path = Win32::GetLongPathName($filename_full_path);
1397
1398 $file = filename_within_directory($long_full_path,$long_base_dir);
1399 $file = encode("utf8",$file) if ($gli);
1400 }
1401
1402 return $file;
1403}
1404
1405
1406sub upgrade_if_dos_filename
1407{
1408 my ($filename_full_path,$and_encode) = @_;
1409
1410 if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
1411 # Ensure any DOS-like filename, such as test~1.txt, has been upgraded
1412 # to its long (Windows) version
1413 my $long_filename = Win32::GetLongPathName($filename_full_path);
1414 if (defined $long_filename) {
1415 $filename_full_path = $long_filename;
1416 }
1417 # Make sure initial drive letter is lower-case (to fit in with rest of Greenstone)
1418 $filename_full_path =~ s/^(.):/\u$1:/;
1419 if ((defined $and_encode) && ($and_encode)) {
1420 $filename_full_path = encode("utf8",$filename_full_path);
1421 }
1422 }
1423
1424 return $filename_full_path;
1425}
1426
1427
1428sub downgrade_if_dos_filename
1429{
1430 my ($filename_full_path) = @_;
1431
1432 if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
1433 require Win32;
1434
1435 # Ensure the given long Windows filename is in a form that can
1436 # be opened by Perl => convert it to a short DOS-like filename
1437
1438 my $short_filename = Win32::GetShortPathName($filename_full_path);
1439 if (defined $short_filename) {
1440 $filename_full_path = $short_filename;
1441 }
1442 # Make sure initial drive letter is lower-case (to fit in
1443 # with rest of Greenstone)
1444 $filename_full_path =~ s/^(.):/\u$1:/;
1445 }
1446
1447 return $filename_full_path;
1448}
1449
1450sub block_filename
1451{
1452 my ($block_hash,$filename) = @_;
1453
1454 if ($ENV{'GSDLOS'} =~ m/^windows$/) {
1455
1456 # lower case the entire thing, eg for cover.jpg when its actually cover.JPG
1457 my $lower_filename = lc($filename);
1458 $block_hash->{'file_blocks'}->{$lower_filename} = 1;
1459# my $lower_drive = $filename;
1460# $lower_drive =~ s/^([A-Z]):/\l$1:/i;
1461
1462# my $upper_drive = $filename;
1463# $upper_drive =~ s/^([A-Z]):/\u$1:/i;
1464#
1465# $block_hash->{'file_blocks'}->{$lower_drive} = 1;
1466# $block_hash->{'file_blocks'}->{$upper_drive} = 1;
1467 }
1468 else {
1469 $block_hash->{'file_blocks'}->{$filename} = 1;
1470 }
1471}
1472
1473
1474sub filename_is_absolute
1475{
1476 my ($filename) = @_;
1477
1478 # support for explicit protocol prefixes, for example file: or hdfs:,
1479 # which must be absolute [jmt12]
1480 if ($filename =~ /^\w+:\/\//)
1481 {
1482 return 1;
1483 }
1484
1485 if ($ENV{'GSDLOS'} =~ /^windows$/i) {
1486 return ($filename =~ m/^(\w:)?\\/);
1487 }
1488 else {
1489 return ($filename =~ m/^\//);
1490 }
1491}
1492
1493
1494## @method make_absolute()
1495#
1496# Ensure the given file path is absolute in respect to the given base path.
1497#
1498# @param $base_dir A string denoting the base path the given dir must be
1499# absolute to.
1500# @param $dir The directory to be made absolute as a string. Note that the
1501# dir may already be absolute, in which case it will remain
1502# unchanged.
1503# @return The now absolute form of the directory as a string.
1504#
1505# @author John Thompson, DL Consulting Ltd.
1506# @copy 2006 DL Consulting Ltd.
1507#
1508#used in buildcol.pl, doesn't work for all cases --kjdon
1509sub make_absolute {
1510
1511 my ($base_dir, $dir) = @_;
1512### print STDERR "dir = $dir\n";
1513 $dir =~ s/[\\\/]+/\//g;
1514 $dir = $base_dir . "/$dir" unless ($dir =~ m|^(\w:)?/|);
1515 $dir =~ s|^/tmp_mnt||;
1516 1 while($dir =~ s|/[^/]*/\.\./|/|g);
1517 $dir =~ s|/[.][.]?/|/|g;
1518 $dir =~ tr|/|/|s;
1519### print STDERR "dir = $dir\n";
1520
1521 return $dir;
1522}
1523## make_absolute() ##
1524
1525sub get_dirsep {
1526
1527 if ($ENV{'GSDLOS'} =~ /^windows$/i) {
1528 return "\\";
1529 } else {
1530 return "\/";
1531 }
1532}
1533
1534sub get_os_dirsep {
1535
1536 if ($ENV{'GSDLOS'} =~ /^windows$/i) {
1537 return "\\\\";
1538 } else {
1539 return "\\\/";
1540 }
1541}
1542
1543sub get_re_dirsep {
1544
1545 return "\\\\|\\\/";
1546}
1547
1548
1549sub get_dirsep_tail {
1550 my ($filename) = @_;
1551
1552 # returns last part of directory or filename
1553 # On unix e.g. a/b.d => b.d
1554 # a/b/c => c
1555
1556 my $dirsep = get_re_dirsep();
1557 my @dirs = split (/$dirsep/, $filename);
1558 my $tail = pop @dirs;
1559
1560 # - caused problems under windows
1561 #my ($tail) = ($filename =~ m/^(?:.*?$dirsep)?(.*?)$/);
1562
1563 return $tail;
1564}
1565
1566
1567# if this is running on windows we want binaries to end in
1568# .exe, otherwise they don't have to end in any extension
1569sub get_os_exe {
1570 return ".exe" if $ENV{'GSDLOS'} =~ /^windows$/i;
1571 return "";
1572}
1573
1574
1575# test to see whether this is a big or little endian machine
1576sub is_little_endian
1577{
1578 # To determine the name of the operating system, the variable $^O is a cheap alternative to pulling it out of the Config module;
1579 # If it is a Macintosh machine (i.e. the Darwin operating system), regardless if it's running on the IBM power-pc cpu or the x86 Intel-based chip with a power-pc emulator running on top of it, it's big-endian
1580 # Otherwise, it's little endian
1581
1582 #return 0 if $^O =~ /^darwin$/i;
1583 #return 0 if $ENV{'GSDLOS'} =~ /^darwin$/i;
1584
1585 # Going back to stating exactly whether the machine is little endian
1586 # or big endian, without any special case for Macs. Since for rata it comes
1587 # back with little endian and for shuttle with bigendian.
1588 return (ord(substr(pack("s",1), 0, 1)) == 1);
1589}
1590
1591
1592# will return the collection name if successful, "" otherwise
1593sub use_collection {
1594 my ($collection, $collectdir) = @_;
1595
1596 if (!defined $collectdir || $collectdir eq "") {
1597 $collectdir = &filename_cat ($ENV{'GSDLHOME'}, "collect");
1598 }
1599
1600 # get and check the collection
1601 if (!defined($collection) || $collection eq "") {
1602 if (defined $ENV{'GSDLCOLLECTION'}) {
1603 $collection = $ENV{'GSDLCOLLECTION'};
1604 } else {
1605 print STDOUT "No collection specified\n";
1606 return "";
1607 }
1608 }
1609
1610 if ($collection eq "modelcol") {
1611 print STDOUT "You can't use modelcol.\n";
1612 return "";
1613 }
1614
1615 # make sure the environment variables GSDLCOLLECTION and GSDLCOLLECTDIR
1616 # are defined
1617 $ENV{'GSDLCOLLECTION'} = $collection;
1618 $ENV{'GSDLCOLLECTDIR'} = &filename_cat ($collectdir, $collection);
1619
1620 # make sure this collection exists
1621 if (!-e $ENV{'GSDLCOLLECTDIR'}) {
1622 print STDOUT "Invalid collection ($collection).\n";
1623 return "";
1624 }
1625
1626 # everything is ready to go
1627 return $collection;
1628}
1629
1630sub get_current_collection_name {
1631 return $ENV{'GSDLCOLLECTION'};
1632}
1633
1634
1635# will return the collection name if successful, "" otherwise.
1636# Like use_collection (above) but for greenstone 3 (taking account of site level)
1637
1638sub use_site_collection {
1639 my ($site, $collection, $collectdir) = @_;
1640
1641 if (!defined $collectdir || $collectdir eq "") {
1642 die "GSDL3HOME not set.\n" unless defined $ENV{'GSDL3HOME'};
1643 $collectdir = &filename_cat ($ENV{'GSDL3HOME'}, "sites", $site, "collect");
1644 }
1645
1646 # collectdir explicitly set by this point (using $site variable if required).
1647 # Can call "old" gsdl2 use_collection now.
1648
1649 return use_collection($collection,$collectdir);
1650}
1651
1652
1653
1654sub locate_config_file
1655{
1656 my ($file) = @_;
1657
1658 my $locations = locate_config_files($file);
1659
1660 return shift @$locations; # returns undef if 'locations' is empty
1661}
1662
1663
1664sub locate_config_files
1665{
1666 my ($file) = @_;
1667
1668 my @locations = ();
1669
1670 if (-e $file) {
1671 # Clearly specified (most likely full filename)
1672 # No need to hunt in 'etc' directories, return value unchanged
1673 push(@locations,$file);
1674 }
1675 else {
1676 # Check for collection specific one before looking in global GSDL 'etc'
1677 if (defined $ENV{'GSDLCOLLECTDIR'} && $ENV{'GSDLCOLLECTDIR'} ne "") {
1678 my $test_collect_etc_filename
1679 = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"etc", $file);
1680
1681 if (-e $test_collect_etc_filename) {
1682 push(@locations,$test_collect_etc_filename);
1683 }
1684 }
1685 my $test_main_etc_filename
1686 = &util::filename_cat($ENV{'GSDLHOME'},"etc", $file);
1687 if (-e $test_main_etc_filename) {
1688 push(@locations,$test_main_etc_filename);
1689 }
1690 }
1691
1692 return \@locations;
1693}
1694
1695
1696sub hyperlink_text
1697{
1698 my ($text) = @_;
1699
1700 $text =~ s/(http:\/\/[^\s]+)/<a href=\"$1\">$1<\/a>/mg;
1701 $text =~ s/(^|\s+)(www\.(\w|\.)+)/<a href=\"http:\/\/$2\">$2<\/a>/mg;
1702
1703 return $text;
1704}
1705
1706
1707# A method to check if a directory is empty (note that an empty directory still has non-zero size!!!)
1708# Code is from http://episteme.arstechnica.com/eve/forums/a/tpc/f/6330927813/m/436007700831
1709sub is_dir_empty
1710{
1711 my ($path) = @_;
1712 opendir DIR, $path;
1713 while(my $entry = readdir DIR) {
1714 next if($entry =~ /^\.\.?$/);
1715 closedir DIR;
1716 return 0;
1717 }
1718 closedir DIR;
1719 return 1;
1720}
1721
1722# Returns the given filename converted using either URL encoding or base64
1723# encoding, as specified by $rename_method. If the given filename has no suffix
1724# (if it is just the tailname), then $no_suffix should be some defined value.
1725# rename_method can be url, none, base64
1726sub rename_file {
1727 my ($filename, $rename_method, $no_suffix) = @_;
1728
1729 if(!$filename) { # undefined or empty string
1730 return $filename;
1731 }
1732
1733 if (!$rename_method) {
1734 print STDERR "WARNING: no file renaming method specified. Defaulting to using URL encoding...\n";
1735 # Debugging information
1736 # my ($cpackage,$cfilename,$cline,$csubr,$chas_args,$cwantarray) = caller(1);
1737 # print STDERR "Called from method: $cfilename:$cline $cpackage->$csubr\n";
1738 $rename_method = "url";
1739 } elsif($rename_method eq "none") {
1740 return $filename; # would have already been renamed
1741 }
1742
1743 # No longer replace spaces with underscores, since underscores mess with incremental rebuild
1744 ### Replace spaces with underscore. Do this first else it can go wrong below when getting tailname
1745 ###$filename =~ s/ /_/g;
1746
1747 my ($tailname,$dirname,$suffix);
1748 if($no_suffix) { # given a tailname, no suffix
1749 ($tailname,$dirname) = File::Basename::fileparse($filename);
1750 }
1751 else {
1752 ($tailname,$dirname,$suffix) = File::Basename::fileparse($filename, "\\.(?:[^\\.]+?)\$");
1753 }
1754 if (!$suffix) {
1755 $suffix = "";
1756 }
1757 else {
1758 $suffix = lc($suffix);
1759 }
1760
1761 if ($rename_method eq "url") {
1762 $tailname = &unicode::url_encode($tailname);
1763 }
1764 elsif ($rename_method eq "base64") {
1765 $tailname = &unicode::base64_encode($tailname);
1766 $tailname =~ s/\s*//sg; # for some reason it adds spaces not just at end but also in middle
1767 }
1768
1769 $filename = "$tailname$suffix";
1770 $filename = "$dirname$filename" if ($dirname ne "./" && $dirname ne ".\\");
1771
1772 return $filename;
1773}
1774
1775
1776# BACKWARDS COMPATIBILITY: Just in case there are old .ldb/.bdb files
1777sub rename_ldb_or_bdb_file {
1778 my ($filename_no_ext) = @_;
1779
1780 my $new_filename = "$filename_no_ext.gdb";
1781 return if (-f $new_filename); # if the file has the right extension, don't need to do anything
1782 # try ldb
1783 my $old_filename = "$filename_no_ext.ldb";
1784
1785 if (-f $old_filename) {
1786 print STDERR "Renaming $old_filename to $new_filename\n";
1787 rename ($old_filename, $new_filename)
1788 || print STDERR "Rename failed: $!\n";
1789 return;
1790 }
1791 # try bdb
1792 $old_filename = "$filename_no_ext.bdb";
1793 if (-f $old_filename) {
1794 print STDERR "Renaming $old_filename to $new_filename\n";
1795 rename ($old_filename, $new_filename)
1796 || print STDERR "Rename failed: $!\n";
1797 return;
1798 }
1799}
1800
1801sub os_dir() {
1802
1803 my $gsdlarch = "";
1804 if(defined $ENV{'GSDLARCH'}) {
1805 $gsdlarch = $ENV{'GSDLARCH'};
1806 }
1807 return $ENV{'GSDLOS'}.$gsdlarch;
1808}
1809
1810# Returns the greenstone URL prefix extracted from the appropriate GS2/GS3 config file.
1811# By default, /greenstone3 for GS3 or /greenstone for GS2.
1812sub get_greenstone_url_prefix() {
1813 # if already set on a previous occasion, just return that
1814 # (Don't want to keep repeating this: cost of re-opening and scanning files.)
1815 return $ENV{'GREENSTONE_URL_PREFIX'} if($ENV{'GREENSTONE_URL_PREFIX'});
1816
1817 my ($configfile, $urlprefix, $defaultUrlprefix);
1818 my @propertynames = ();
1819
1820 if($ENV{'GSDL3SRCHOME'}) {
1821 $defaultUrlprefix = "/greenstone3";
1822 $configfile = &util::filename_cat($ENV{'GSDL3SRCHOME'}, "packages", "tomcat", "conf", "Catalina", "localhost", "greenstone3.xml");
1823 push(@propertynames, qw/path\s*\=/);
1824 } else {
1825 $defaultUrlprefix = "/greenstone";
1826 $configfile = &util::filename_cat($ENV{'GSDLHOME'}, "cgi-bin", &os_dir(), "gsdlsite.cfg");
1827 push(@propertynames, (qw/\nhttpprefix/, qw/\ngwcgi/)); # inspect one property then the other
1828 }
1829
1830 $urlprefix = &extract_propvalue_from_file($configfile, \@propertynames);
1831
1832 if(!$urlprefix) { # no values found for URL prefix, use default values
1833 $urlprefix = $defaultUrlprefix;
1834 } else {
1835 #gwcgi can contain more than the wanted prefix, we split on / to get the first "directory" level
1836 $urlprefix =~ s/^\///; # remove the starting slash
1837 my @dirs = split(/(\\|\/)/, $urlprefix);
1838 $urlprefix = shift(@dirs);
1839
1840 if($urlprefix !~ m/^\//) { # in all cases: ensure the required forward slash is at the front
1841 $urlprefix = "/$urlprefix";
1842 }
1843 }
1844
1845 # set for the future
1846 $ENV{'GREENSTONE_URL_PREFIX'} = $urlprefix;
1847# print STDERR "*** in get_greenstone_url_prefix(): $urlprefix\n\n";
1848 return $urlprefix;
1849}
1850
1851
1852# Given a config file (xml or java properties file) and a list/array of regular expressions
1853# that represent property names to match on, this function will return the value for the 1st
1854# matching property name. If the return value is undefined, no matching property was found.
1855sub extract_propvalue_from_file() {
1856 my ($configfile, $propertynames) = @_;
1857
1858 my $value;
1859 unless(open(FIN, "<$configfile")) {
1860 print STDERR "extract_propvalue_from_file(): Unable to open $configfile. $!\n";
1861 return $value; # not initialised
1862 }
1863
1864 # Read the entire file at once, as one single line, then close it
1865 my $filecontents;
1866 {
1867 local $/ = undef;
1868 $filecontents = <FIN>;
1869 }
1870 close(FIN);
1871
1872 foreach my $regex (@$propertynames) {
1873 ($value) = $filecontents=~ m/$regex\s*(\S*)/s; # read value of the property given by regex up to the 1st space
1874 if($value) {
1875 $value =~ s/^\"//; # remove any startquotes
1876 $value =~ s/\".*$//; # remove the 1st endquotes (if any) followed by any xml
1877 last; # found value for a matching property, break from loop
1878 }
1879 }
1880
1881 return $value;
1882}
1883
1884# Subroutine that sources setup.bash, given GSDLHOME and GSDLOS and
1885# given that perllib is in @INC in order to invoke this subroutine.
1886# Call as follows -- after setting up INC to include perllib and
1887# after setting up GSDLHOME and GSDLOS:
1888#
1889# require util;
1890# &util::setup_greenstone_env($ENV{'GSDLHOME'}, $ENV{'GSDLOS'});
1891#
1892sub setup_greenstone_env() {
1893 my ($GSDLHOME, $GSDLOS) = @_;
1894
1895 #my %env_map = ();
1896 # Get the localised ENV settings of running a localised source setup.bash
1897 # and put it into the ENV here. Need to clear GSDLHOME before running setup
1898 #my $perl_command = "(cd $GSDLHOME; export GSDLHOME=; . ./setup.bash > /dev/null; env)";
1899 my $perl_command = "(cd $GSDLHOME; /bin/bash -c \"export GSDLHOME=; source setup.bash > /dev/null; env\")";
1900 if($GSDLOS =~ m/windows/i) {
1901 #$perl_command = "cmd /C \"cd $GSDLHOME&& set GSDLHOME=&& setup.bat > nul&& set\"";
1902 $perl_command = "(cd $GSDLHOME&& set GSDLHOME=&& setup.bat > nul&& set)";
1903 }
1904 if (!open(PIN, "$perl_command |")) {
1905 print STDERR ("Unable to execute command: $perl_command. $!\n");
1906 }
1907
1908 while (defined (my $perl_output_line = <PIN>)) {
1909 my($key,$value) = ($perl_output_line =~ m/^([^=]*)[=](.*)$/);
1910 #$env_map{$key}=$value;
1911 $ENV{$key}=$value;
1912 }
1913 close (PIN);
1914
1915 # If any keys in $ENV don't occur in Greenstone's localised env
1916 # (stored in $env_map), delete those entries from $ENV
1917 #foreach $key (keys %ENV) {
1918 # if(!defined $env_map{$key}) {
1919 # print STDOUT "**** DELETING ENV KEY: $key\tVALUE: $ENV{'$key'}\n";
1920 # delete $ENV{$key}; # del $ENV(key, value) pair
1921 # }
1922 #}
1923 #undef %env_map;
1924}
1925
1926sub get_perl_exec() {
1927 my $perl_exec = $^X; # may return just "perl"
1928
1929 if($ENV{'PERLPATH'}) {
1930 # OR: # $perl_exec = &util::filename_cat($ENV{'PERLPATH'},"perl");
1931 if($ENV{'GSDLOS'} =~ m/windows/) {
1932 $perl_exec = "$ENV{'PERLPATH'}\\Perl.exe";
1933 } else {
1934 $perl_exec = "$ENV{'PERLPATH'}/perl";
1935 }
1936 } else { # no PERLPATH, use Config{perlpath} else $^X: special variables
1937 # containing the full path to the current perl executable we're using
1938 $perl_exec = $Config{perlpath}; # configured path for perl
1939 if (!-e $perl_exec) { # may not point to location on this machine
1940 $perl_exec = $^X; # may return just "perl"
1941 if($perl_exec =~ m/^perl/i) { # warn if just perl or Perl.exe
1942 print STDERR "**** WARNING: Perl exec found contains no path: $perl_exec";
1943 }
1944 }
1945 }
1946
1947 return $perl_exec;
1948}
1949
1950# returns the path to the java command in the JRE included with GS (if any),
1951# quoted to safeguard any spaces in this path, otherwise a simple java
1952# command is returned which assumes and will try for a system java.
1953sub get_java_command {
1954 my $java = "java";
1955 if(defined $ENV{'GSDLHOME'}) { # should be, as this script would be launched from the cmd line
1956 # after running setup.bat or from GLI which also runs setup.bat
1957 my $java_bin = &util::filename_cat($ENV{'GSDLHOME'},"packages","jre","bin");
1958 if(-d $java_bin) {
1959 $java = &util::filename_cat($java_bin,"java");
1960 $java = "\"".$java."\""; # quoted to preserve spaces in path
1961 }
1962 }
1963 return $java;
1964}
1965
1966
1967# Given the qualified collection name (colgroup/collection),
1968# returns the collection and colgroup parts
1969sub get_collection_parts {
1970 # http://perldoc.perl.org/File/Basename.html
1971 # my($filename, $directories, $suffix) = fileparse($path);
1972 # "$directories contains everything up to and including the last directory separator in the $path
1973 # including the volume (if applicable). The remainder of the $path is the $filename."
1974 #my ($collection, $colgroup) = &File::Basename::fileparse($qualified_collection);
1975
1976 my $qualified_collection = shift(@_);
1977
1978 # Since activate.pl can be launched from the command-line, including by a user,
1979 # best not to assume colgroup uses URL-style slashes as would be the case with GLI
1980 # Also allow for the accidental inclusion of multiple slashes
1981 my ($colgroup, $collection) = split(/[\/\\]+/, $qualified_collection); #split('/', $qualified_collection);
1982
1983 if(!defined $collection) {
1984 $collection = $colgroup;
1985 $colgroup = "";
1986 }
1987 return ($collection, $colgroup);
1988}
1989
1990# work out the "collectdir/collection" location
1991sub resolve_collection_dir {
1992 my ($collect_dir, $qualified_collection, $site) = @_; #, $gs_mode
1993
1994 my ($colgroup, $collection) = &util::get_collection_parts($qualified_collection);
1995
1996 if (defined $collect_dir) {
1997 return &util::filename_cat($collect_dir,$colgroup, $collection);
1998 }
1999 else {
2000 if (defined $site) {
2001 return &util::filename_cat($ENV{'GSDL3HOME'},"sites",$site,"collect",$colgroup, $collection);
2002 }
2003 else {
2004 return &util::filename_cat($ENV{'GSDLHOME'},"collect",$colgroup, $collection);
2005 }
2006 }
2007}
2008
2009# ====== John's new fuctions to support HDFS ======
2010
2011# /**
2012# * Executes a HDFS command without caring about the resulting output while
2013# * still reacting appropriately to failed executions.
2014# */
2015sub executeHDFSCommand
2016{
2017 my $action = shift(@_);
2018 my $command = 'hadoop fs -' . $action . ' "' . join('" "', @_) . '"';
2019 my $result = `$command 2>&1`;
2020 my $return_value = $?;
2021 ###rint STDERR "-> util::executeHDFSCommand('" . $command . "') => " . $return_value . "\n";
2022 return $return_value;
2023}
2024
2025# /** @function file_canread()
2026# */
2027sub file_canread
2028{
2029 my ($filename_full_path) = @_;
2030 # the HDFS support doesn't have '-r' so it will revert to '-e'
2031 return fd_exists($filename_full_path,"-r");
2032}
2033# /** file_canread() **/
2034
2035sub file_openfdcommand
2036{
2037 my ($filename_full_path, $mode) = @_;
2038 # I'll set to read by default, as that is less destructive to precious files
2039 # on your system...
2040 if (!defined $mode)
2041 {
2042 $mode = '<';
2043 }
2044 my $open_fd_command = $mode . $filename_full_path;
2045 if (&util::isHDFS($filename_full_path))
2046 {
2047 # currently don't really support append, but might be able to do something
2048 # like:
2049 # hadoop fs -cat /user/username/folder/csv1.csv \
2050 # /user/username/folder/csv2.csv | hadoop fs -put - \
2051 # /user/username/folder/output.csv
2052 if ($mode eq '>>' || $mode eq '>')
2053 {
2054 # if the file already exists, put won't clobber it like a proper write
2055 # would, so try to delete it (can fail)
2056 if (&util::file_exists($filename_full_path))
2057 {
2058 &util::rm($filename_full_path);
2059 }
2060 # then create the command
2061 $open_fd_command = '| ' . &util::generateHDFSCommand('put', '-', $filename_full_path);
2062 }
2063 else
2064 {
2065 $open_fd_command = &util::generateHDFSCommand('cat', $filename_full_path) . ' |';
2066 }
2067 }
2068 return $open_fd_command;
2069}
2070
2071# /** @function file_readdir()
2072# * Provide a function to return the files within a directory that is aware
2073# * of protocols other than file://
2074# * @param $dirname the full path to the directory
2075# * @param $dir_ref a reference to an array to populate with files
2076# */
2077sub file_readdir
2078{
2079 my ($dirname, $dir_ref) = @_;
2080 my $dir_read = 0;
2081 if (&util::isHDFS($dirname))
2082 {
2083 my $hdfs_command = &util::generateHDFSCommand('ls', $dirname);
2084 my $result = `$hdfs_command 2>&1`;
2085 my @lines = split(/\r?\n/, $result);
2086 foreach my $line (@lines)
2087 {
2088 if ($line =~ /\/([^\/]+)$/)
2089 {
2090 my $file = $1;
2091 push(@{$dir_ref}, $file);
2092 }
2093 }
2094 $dir_read = 1;
2095 }
2096 elsif (opendir(DIR, $dirname))
2097 {
2098 my @dirs = readdir(DIR);
2099 push(@{$dir_ref}, @dirs);
2100 closedir(DIR);
2101 $dir_read = 1;
2102 }
2103 return $dir_read;
2104}
2105# /** file_readdir() **/
2106
2107# /**
2108# */
2109sub file_lastmodified
2110{
2111 my ($filename_full_path) = @_;
2112 my $last_modified = 0;
2113 if (&util::isHDFS($filename_full_path))
2114 {
2115 my $file_stats = file_stats($filename_full_path);
2116 my $mod_date = $file_stats->{'modification_date'};
2117 my $mod_time = $file_stats->{'modification_time'};
2118 # Last modified should be in number of days (as a float) since last
2119 # modified - but I'll just return 0 for now
2120 $last_modified = 0.00;
2121 }
2122 else
2123 {
2124 $last_modified = -M $filename_full_path;
2125 }
2126 return $last_modified;
2127}
2128# /** file_lastmodified() **/
2129
2130# /** @function file_size
2131# * Replacement for "-s" in Greenstone buildtime, as we need a version that
2132# * HDFS aware
2133# */
2134sub file_size
2135{
2136 my ($filename_full_path) = @_;
2137 my $size = 0;
2138 if (&util::isHDFS($filename_full_path))
2139 {
2140 my $file_stats = file_stats($filename_full_path);
2141 $size = $file_stats->{'filesize'};
2142 }
2143 else
2144 {
2145 $size = -s $filename_full_path;
2146 }
2147 return $size;
2148}
2149# /** file_size() **/
2150
2151sub file_stats
2152{
2153 my ($filename_full_path) = @_;
2154 my $stats = {};
2155 if (&util::isHDFS($filename_full_path))
2156 {
2157 # - LS is the only way to get these details from HDFS (-stat doesn't
2158 # provide enough information)
2159 my $hdfs_command = &util::generateHDFSCommand('ls', $filename_full_path);
2160 my $result = `$hdfs_command 2>&1`;
2161 # - parse the results
2162 if ($result =~ /([ds\-][rwx\-]+)\s+(\d+)\s+([^\s]+)\s+([^\s]+)\s+(\d+)\s+(\d\d\d\d-\d\d-\d\d)\s+(\d\d:\d\d)\s+([^\s]+)$/)
2163 {
2164 $stats->{'filename'} = $8;
2165 $stats->{'replicas'} = $2;
2166 $stats->{'filesize'} = $5;
2167 $stats->{'modification_date'} = $6;
2168 $stats->{'modification_time'} = $7;
2169 $stats->{'permissions'} = $1;
2170 $stats->{'userid'} = $3;
2171 $stats->{'groupid'} = $4;
2172 }
2173 else
2174 {
2175 die("Error! Failed to parse HDFS ls result: " . $result . "\n");
2176 }
2177 }
2178 return $stats;
2179}
2180# /** file_stats() **/
2181
2182# /**
2183# */
2184sub generateHDFSCommand
2185{
2186 my $flags = shift(@_);
2187 return 'hadoop fs -' . $flags . ' "' . join('" "', @_) . '"';
2188}
2189# /** generateHDFSCommand() **/
2190
2191# /** @function isHDFS()
2192# * Determine if the given path exists within a HDFS system by checking for
2193# * the expected protocol prefix.
2194# * @param $full_path the path to check
2195# * @return 1 if within HDFS, 0 otherwise
2196# */
2197sub isHDFS
2198{
2199 my ($full_path) = @_;
2200 return (lc(substr($full_path, 0, 7)) eq 'hdfs://');
2201}
2202# /** isHDFS() **/
2203
22041;
Note: See TracBrowser for help on using the repository browser.