Changeset 28355 for main/trunk/greenstone2

Timestamp:

2013-10-04T19:52:18+13:00 (11 years ago)

Author:

ak19

Message:

1. Now gsConvert.pl calls the new pptextract.vbs VBScript (which creates .item files and ppt slide.txt files in utf-8) instead of the older VB pptextract.exe executable which created .item and slide.txt files in windows default utf-16 LE. 2. PagedImagePlugin.pm::tidy_item_file now reads in the .item files in utf-8 mode, so that its strings are unicode aware. Substitutions are of unicode code points instead of byte sequences, since the strings in the file are now unicode aware.

Location:

main/trunk/greenstone2

Files:

• bin/script/gsConvert.pl (modified) (3 diffs)
• perllib/plugins/PagedImagePlugin.pm (modified) (2 diffs)

main/trunk/greenstone2/bin/script/gsConvert.pl

@@ -386 +386 @@
     my $vbScript = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin",
                        $ENV{'GSDLOS'}, "pptextract");
-    $vbScript = "pptextract" if ($ENV{'GSDLOS'} =~ m/^windows$/i);
+    $vbScript = "CScript //Nologo \"".$vbScript.".vbs\"" if ($ENV{'GSDLOS'} =~ m/^windows$/i); # now we use the .vbs VBScript
+    # $vbScript = "pptextract" if ($ENV{'GSDLOS'} =~ m/^windows$/i); # back when the pptextract.exe VB executable was used
             
     my $cmd = "";
@@ -398 +399 @@
         $cmd .= " 2>\"$output_filestem.err\""
         if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000);
+
         if (system($cmd) !=0) {
         print STDERR "Powerpoint VB Scripting convert failed\n";
@@ -549 +551 @@
                                             # else script launch fails when there are error msgs
             $vbScript = &FileUtils::filenameConcatenate($vbScript, "docx2html.vbs"); 
-            $vbScript = "CScript //Nologo \"$vbScript\"";   # launche with CScript for error output in STDERR
+            $vbScript = "CScript //Nologo \"$vbScript\"";   # launch with CScript for error output in STDERR
                                     # //Nologo flag avoids Microsoft's opening/logo msgs
             print STDERR "About to use windows scripting to process docx file $input_filename.\n";

main/trunk/greenstone2/perllib/plugins/PagedImagePlugin.pm

@@ -407 +407 @@
     my ($filename) = @_;
 
-    open (ITEMFILE, $filename) || die "couldn't open $filename\n";
+    open (ITEMFILE, "<:encoding(UTF-8)", $filename) || die "couldn't open $filename\n";
     my $backup_filename = "backup.item";
     open (BACKUP,">$backup_filename")|| die "couldn't write to $backup_filename\n";
+    binmode(BACKUP, ":utf8");
     my $line = "";
     $line = <ITEMFILE>;
-    $line =~ s/^\xEF\xBB\xBF//; # strip BOM
-    $line =~ s/\x0B+//ig;
+    #$line =~ s/^\xEF\xBB\xBF//; # strip BOM in text file read in as a sequence of bytes (not unicode aware strings)
+    $line =~ s/^\x{FEFF}//; # strip BOM in file opened *as UTF-8*. Strings in the file just read in are now unicode-aware,
+                            # this means the BOM is now a unicode codepoint instead of a byte sequence
+                            # See http://en.wikipedia.org/wiki/Byte_order_mark and http://perldoc.perl.org/5.14.0/perlunicode.html 
+    $line =~ s/\x{0B}+//ig; # removing \vt-vertical tabs using the unicode codepoint for \vt
     $line =~ s/&/&amp;/g;
     print BACKUP ($line);
     #Tidy up the item file some metadata title contains \vt-vertical tab
     while ($line = <ITEMFILE>) {
-    $line =~ s/\x0B+//ig;
+    $line =~ s/\x{0B}+//ig; # removing \vt-vertical tabs using the unicode codepoint for \vt
     $line =~ s/&/&amp;/g;
     print BACKUP ($line);
@@ -657 +661 @@
     my $self = shift (@_);
     my ($filename_full_path, $dir, $block_hash) = @_;
+
 
     open (ITEMFILE, $filename_full_path) || die "couldn't open $filename_full_path to work out which files to block\n";