Changeset 27377 for gs2-extensions/parallel-building/trunk/src
- Timestamp:
- 2013-05-21T14:46:55+12:00 (11 years ago)
- Location:
- gs2-extensions/parallel-building/trunk/src/perllib
- Files:
-
- 4 edited
Legend:
- Unmodified
- Added
- Removed
-
gs2-extensions/parallel-building/trunk/src/perllib/plugins/DirectoryPlugin.pm
r26987 r27377 181 181 182 182 # replace -d with function in util library 183 return undef unless (& util::dir_exists($dirname));183 return undef unless (&FileUtils::directoryExists($dirname)); 184 184 185 185 return 0 if ($self->{'block_exp'} ne "" && $dirname =~ /$self->{'block_exp'}/); … … 308 308 $block_hash->{'shared_fileroot'} = {} unless defined $block_hash->{'shared_fileroot'}; 309 309 310 # Recur over directory contents. 311 my (@dir, $subfile); 312 #my $count = 0; 313 310 # Recur over directory contents. 314 311 print $outhandle "DirectoryPlugin block: getting directory $dirname\n" if ($verbosity > 2); 315 312 316 313 # find all the files in the directory 317 my $dir_read = &util::file_readdir($dirname, \@dir); 318 if (!$dir_read) { 319 if ($gli) { 320 print STDERR "<ProcessingError n='$file' r='Could not read directory $dirname'>\n"; 321 } 322 print $outhandle "DirectoryPlugin: WARNING - couldn't read directory $dirname\n"; 323 return -1; # error in processing 314 my @dir = @{&FileUtils::readDirectory($dirname)}; 315 if (scalar(@dir) == 0) 316 { 317 if ($gli) 318 { 319 print STDERR "<ProcessingError n='$file' r='Could not read directory $dirname'>\n"; 320 } 321 print $outhandle "DirectoryPlugin: WARNING - couldn't read directory $dirname\n"; 322 return -1; # error in processing 324 323 } 325 324 … … 394 393 395 394 # Recur over directory contents. 396 my (@dir, $subfile);397 395 print $outhandle "DirectoryPlugin read: getting directory $dirname\n" if ($verbosity > 2); 398 396 399 397 # find all the files in the directory 400 my $dir_read = &util::file_readdir($dirname, \@dir);401 if ( !$dir_read)398 my @dir = @{&FileUtils::readDirectory($dirname)}; 399 if (scalar(@dir) == 0) 402 400 { 403 401 if ($gli) -
gs2-extensions/parallel-building/trunk/src/perllib/plugins/ReadTextFile.pm
r26963 r27377 39 39 use ghtml; 40 40 use gsprintf 'gsprintf'; 41 use util; 42 use FileUtils; 41 43 42 44 use AutoExtractMetadata; … … 140 142 # this should look at the plugin option too... 141 143 $doc_obj->add_utf8_metadata($top_section, "Plugin", "$self->{'plugin_type'}"); 142 $doc_obj->add_utf8_metadata($top_section, "FileSize", & util::file_size($filename_full_path));144 $doc_obj->add_utf8_metadata($top_section, "FileSize", &FileUtils::fileSize($filename_full_path)); 143 145 144 146 my $plugin_filename_encoding = $self->{'filename_encoding'}; … … 202 204 my ($filename, $encoding, $language, $textref) = @_; 203 205 204 if (!& util::file_canread($filename))206 if (!&FileUtils::canRead($filename)) 205 207 { 206 208 my $outhandle = $self->{'outhandle'}; … … 213 215 # use the util::file_* functions to provide some extra smarts to the open 214 216 # command in order to support protocols like http: or hdfs: 215 if (!open (FILE, &util::file_openfdcommand($filename, '<'))) 217 my $fh; 218 if (!&FileUtils::openFileHandle($filename, '<', \$fh)) 216 219 { 217 220 gsprintf(STDERR, "ReadTextFile::read_file {ReadTextFile.could_not_open_for_reading} ($!)\n", $filename); … … 222 225 { 223 226 # fast file 'slurp' implementation 224 sysread( FILE, $$textref, &util::file_size($filename));227 sysread($fh, $$textref, &FileUtils::fileSize($filename)); 225 228 # the old slow way of reading in a file 226 229 #undef $/; … … 231 234 { 232 235 my $reader = new multiread(); 233 $reader->set_handle ( 'ReadTextFile::FILE');236 $reader->set_handle ($fh); 234 237 $reader->set_encoding ($encoding); 235 238 $reader->read_file ($textref); … … 243 246 $$textref = decode("utf8", $$textref); 244 247 245 close FILE;248 &FileUtils::closeFileHandle($filename, \$fh); 246 249 } 247 250 … … 368 371 my $best_language = ""; 369 372 my $best_encoding = ""; 370 371 373 372 374 # read in file 373 if (!open (FILE, &util::file_openfdcommand($filename, '<'))) { 374 gsprintf(STDERR, "ReadTextFile::get_language_encoding {ReadTextFile.could_not_open_for_reading} ($!)\n", $filename); 375 # this is a pretty bad error, but try to continue anyway 376 return ($self->{'default_language'}, $self->{'input_encoding'}); 375 my $fh; 376 if (!&FileUtils::openFileHandle($filename, '<', \$fh)) 377 { 378 gsprintf(STDERR, "ReadTextFile::get_language_encoding {ReadTextFile.could_not_open_for_reading} ($!)\n", $filename); 379 # this is a pretty bad error, but try to continue anyway 380 return ($self->{'default_language'}, $self->{'input_encoding'}); 377 381 } 378 382 my $text; 379 sysread( FILE, $text, &util::file_size($filename));383 sysread($fh, $text, &FileUtils::fileSize($filename)); 380 384 #undef $/; 381 385 #my $text = <FILE>; 382 386 #$/ = "\n"; 383 close FILE;387 close $fh; 384 388 385 389 # check if first few bytes have a Byte Order Marker -
gs2-extensions/parallel-building/trunk/src/perllib/plugouts/BasePlugout.pm
r26964 r27377 3 3 # BasePlugout.pm -- base class for all the plugout modules 4 4 # A component of the Greenstone digital library software 5 # from the New Zealand Digital Library Project at the 5 # from the New Zealand Digital Library Project at the 6 6 # University of Waikato, New Zealand. 7 7 # … … 36 36 use printusage; 37 37 use parse2; 38 use util; 39 use FileUtils; 38 40 39 41 … … 349 351 350 352 my ($output_file_name) = @_; 351 352 open(*OUTPUT, ">$output_file_name") or die "Can not open a file handler for $output_file_name\n"; 353 354 return *OUTPUT; 353 my $fh; 354 &FileUtils::openFileHandle($output_file_name, '>', \$fh) or die "Can not open a file handler for $output_file_name\n"; 355 return $fh; 355 356 } 356 357 … … 508 509 # only if opening new file 509 510 my $output_dir = $self->get_output_dir(); 510 & util::mk_all_dir ($output_dir) unless &util::dir_exists($output_dir);511 my $doc_file = & util::filename_cat($output_dir, $doc_dir, "doc.xml");512 my $short_doc_file = & util::filename_cat($doc_dir, "doc.xml");511 &FileUtils::makeAllDirectories ($output_dir) unless &FileUtils::directoryExists($output_dir); 512 my $doc_file = &FileUtils::filenameConcatenate ($output_dir, $doc_dir, "doc.xml"); 513 my $short_doc_file = &FileUtils::filenameConcatenate ($doc_dir, "doc.xml"); 513 514 514 515 if ($gs_count>0) … … 590 591 591 592 if (!defined $self->{'group'} || !$self->{'group'}){ 592 & util::mk_all_dir (&util::filename_cat($working_dir, $doc_dir));593 &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir)); 593 594 } 594 595 … … 619 620 $doc_dir_num++; 620 621 } 621 } while ($doc_dir_rest ne "" && ((& util::dir_exists(&util::filename_cat($working_dir, "$doc_dir.dir"))) || ($working_info->size() >= 1024 && $doc_dir_num < 2)));622 } while ($doc_dir_rest ne "" && ((&FileUtils::directoryExists(&FileUtils::filenameConcatenate($working_dir, "$doc_dir.dir"))) || ($working_info->size() >= 1024 && $doc_dir_num < 2))); 622 623 my $i = 1; 623 624 my $doc_dir_base = $doc_dir; 624 625 #rint "!! - ensure path is unique by adding numeric suffix if necessary\n"; 625 while (& util::dir_exists(&util::filename_cat($working_dir, "$doc_dir.dir"))) {626 while (&FileUtils::directoryExists(&FileUtils::filenameConcatenate($working_dir, "$doc_dir.dir"))) { 626 627 $doc_dir = "$doc_dir_base-$i"; 627 628 $i++; … … 640 641 return if (!defined $output_dir); 641 642 642 & util::mk_all_dir ($output_dir) unless &util::dir_exists($output_dir);643 &FileUtils::makeAllDirectories($output_dir) unless &FileUtils::directoryExists($output_dir); 643 644 644 my $working_dir = & util::filename_cat($output_dir, $doc_dir);645 & util::mk_all_dir ($working_dir) unless &util::dir_exists($working_dir);645 my $working_dir = &FileUtils::filenameConcatenate($output_dir, $doc_dir); 646 &FileUtils::makeAllDirectories($working_dir) unless &FileUtils::directoryExists($working_dir); 646 647 647 648 my @assoc_files = (); … … 662 663 663 664 if ($source_filename !~ /^$dirsep_regexp/) { 664 $source_filename = & util::filename_cat($collect_dir, $source_filename);665 $source_filename = &FileUtils::filenameConcatenate($collect_dir, $source_filename); 665 666 } 666 667 } … … 684 685 $real_filename = &util::downgrade_if_dos_filename($real_filename); 685 686 686 if (& util::file_exists($real_filename)) {687 688 $filename = & util::filename_cat($working_dir, $afile);689 690 & util::hard_link($real_filename, $filename, $self->{'verbosity'});687 if (&FileUtils::fileExists($real_filename)) { 688 689 $filename = &FileUtils::filenameConcatenate($working_dir, $afile); 690 691 &FileUtils::hardLink($real_filename, $filename, $self->{'verbosity'}); 691 692 692 693 $doc_obj->add_utf8_metadata ($doc_obj->get_top_section(), … … 731 732 my $raw_filename = &util::downgrade_if_dos_filename($real_filename); 732 733 733 if (& util::file_exists($raw_filename)) {734 if (&FileUtils::fileExists($raw_filename)) { 734 735 735 736 # if (defined $collect_dir) { … … 819 820 $dc_title .= " (Updated)"; 820 821 } 821 my $rss_filename = &util::filename_cat($output_dir,"rss-items.rdf"); 822 if (open(RSSOUT, &util::file_openfdcommand($rss_filename, '>>'))) 822 my $rss_filename = &FileUtils::filenameConcatenate($output_dir,"rss-items.rdf"); 823 my $rssout_fh; 824 825 if (&FileUtils::openFileHandle($rss_filename, '>>', \$rssout_fh)) 823 826 { 824 print RSSOUT"<item>\n";825 print RSSOUT" <title>" . $dc_title . "</title>\n";826 print RSSOUT" <link>_httpdomain__httpcollection_/document/" . $doc_obj->get_OID() . "</link>\n";827 print RSSOUT"</item>\n";828 close(RSSOUT);827 print $rssout_fh "<item>\n"; 828 print $rssout_fh " <title>" . $dc_title . "</title>\n"; 829 print $rssout_fh " <link>_httpdomain__httpcollection_/document/" . $doc_obj->get_OID() . "</link>\n"; 830 print $rssout_fh "</item>\n"; 831 &FileUtils::closeFileHandle($rss_filename, $rssout_fh); 829 832 } 830 833 else … … 887 890 my ($output_file_name, $xslt_file)=@_; 888 891 889 return unless defined $xslt_file and $xslt_file ne "" and & util::file_exists($xslt_file);890 891 my $java_class_path = & util::filename_cat($ENV{'GSDLHOME'},"bin","java","ApplyXSLT.jar");892 return unless defined $xslt_file and $xslt_file ne "" and &FileUtils::fileExists($xslt_file); 893 894 my $java_class_path = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'},"bin","java","ApplyXSLT.jar"); 892 895 893 896 my $mapping_file_path = ""; 894 897 895 898 if ($ENV{'GSDLOS'} eq "windows"){ 896 $java_class_path .=";".& util::filename_cat($ENV{'GSDLHOME'},"bin","java","xalan.jar");899 $java_class_path .=";".&FileUtils::filenameConcatenate($ENV{'GSDLHOME'},"bin","java","xalan.jar"); 897 900 # this file:/// bit didn't work for me on windows XP 898 901 #$xslt_file = "\"file:///".$xslt_file."\""; … … 900 903 } 901 904 else{ 902 $java_class_path .=":".& util::filename_cat($ENV{'GSDLHOME'},"bin","java","xalan.jar");905 $java_class_path .=":".&FileUtils::filenameConcatenate($ENV{'GSDLHOME'},"bin","java","xalan.jar"); 903 906 } 904 907 … … 961 964 $doc_file .= ".gz"; 962 965 $short_doc_file .= ".gz"; 963 if (!& util::file_exists($doc_file)) {966 if (!&FileUtils::fileExists($doc_file)) { 964 967 my $outhandle = $self->{'output_handle'}; 965 968 print $outhandle "error while gzipping: $doc_file doesn't exist\n"; -
gs2-extensions/parallel-building/trunk/src/perllib/plugouts/GreenstoneXMLPlugout.pm
r26986 r27377 32 32 eval {require bytes}; 33 33 use util; 34 use FileUtils; 34 35 use BasePlugout; 35 36 use docprint; … … 73 74 { 74 75 my $output_dir = $self->get_output_dir(); 75 my $working_dir = & util::filename_cat($output_dir, $doc_dir);76 if (!& util::dir_exists($working_dir))76 my $working_dir = &FileUtils::filenameConcatenate($output_dir, $doc_dir); 77 if (!&FileUtils::directoryExists($working_dir)) 77 78 { 78 & util::mk_all_dir($working_dir);79 &FileUtils::makeAllDirectories($working_dir); 79 80 } 80 81 $self->process_assoc_files ($doc_obj, $doc_dir, ''); 81 82 $self->process_metafiles_metadata ($doc_obj); 82 my $output_file = & util::filename_cat($working_dir, "doc.xml");83 my $output_file = &FileUtils::filenameConcatenate($working_dir, "doc.xml"); 83 84 $self->open_xslt_pipe($output_file, $self->{'xslt_file'}); 84 85 if (defined $self->{'xslt_writer'}) … … 110 111 } 111 112 112 $self->{'short_doc_file'} = util::filename_cat($doc_dir, "doc.xml");113 $self->{'short_doc_file'} = FileUtils::filenameConcatenate($doc_dir, "doc.xml"); 113 114 114 115 $self->store_output_info_reference($doc_obj); … … 116 117 } 117 118 # /** saveas() **/ 118 119 # Overriden for HDFS support120 sub get_output_handler121 {122 my $self = shift (@_);123 my ($output_file_name) = @_;124 if (&util::isHDFS($output_file_name))125 {126 open(*OUTPUT, &util::file_openfdcommand($output_file_name, '>'));127 return *OUTPUT;128 }129 else130 {131 return $self->SUPER::get_output_handler($output_file_name);132 }133 }134 119 135 120 sub get_new_doc_dir … … 159 144 $doc_dir_num++; 160 145 } 161 # ##rint "!! - testing Path: " . $doc_dir . "\n";162 $created_directory = & util::mk_all_dir(&util::filename_cat($working_dir, $doc_dir . '.dir'));163 # ##rint "-> result: |" . $created_directory . "|\n";146 #rint "!! - testing Path: " . $doc_dir . "\n"; 147 $created_directory = &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir')); 148 #rint "-> result: |" . $created_directory . "|\n"; 164 149 } 165 150 while ($doc_dir_rest ne "" && $doc_dir_num < 32 && !$created_directory); … … 170 155 $i++; 171 156 $doc_dir = $doc_dir_base . '-' . $i; 172 # ##rint "!! - testing Path: " . $doc_dir . "\n";173 $created_directory = & util::mk_all_dir(&util::filename_cat($working_dir, $doc_dir . '.dir'));174 # ##rint "-> result: |" . $created_directory . "|\n";157 #rint "!! - testing Path: " . $doc_dir . "\n"; 158 $created_directory = &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir')); 159 #rint "-> result: |" . $created_directory . "|\n"; 175 160 } 176 161 if (!$created_directory) … … 178 163 die("Error! Failed to create directory for document: " . $doc_dir_base . "\n"); 179 164 } 180 # ##rint "!! Final Path: " . $doc_dir . ".dir\n";165 #rint "!! Final Path: " . $doc_dir . ".dir\n"; 181 166 return $doc_dir . '.dir'; 182 167 }
Note:
See TracChangeset
for help on using the changeset viewer.