Changeset 33717

Timestamp:

2019-11-22T10:44:13+13:00 (4 years ago)

Author:

davidb

Message:

Code tidy up; better error checking on running Java cmd; public/private key for TotalVirus added; sleep added if public key

Location:

gs2-extensions/malware-checker/trunk/perllib/plugins

Files:

• MalwareCheckerConverter.pm (modified) (1 diff)
• PDFv3Plugin.pm (modified) (2 diffs)

gs2-extensions/malware-checker/trunk/perllib/plugins/MalwareCheckerConverter.pm

@@ -179 +179 @@
 
 
-sub convert {
-    my $self = shift(@_);
-    my ($source_file_full_path, $target_file_type) = @_;
-
-    return 0 unless $malwarechecker_conversion_available;
-    # check the filename
-    return 0 if ( !-f $source_file_full_path);
-
-    # Although PDFBoxConverter inherits from AutoLoadConverters and therefore
-    # doesn't go through gsConvert.pl, still set the -pdf_tool flag in convert_options
-    # in case in future PDFBoxConverter no longer inherits from AutoLoadConverters
-    # and ends up going through gsConvert.pl
-    $self->{'convert_options'} .= " -pdf_tool pdfbox";
-
-    my $img_output_mode = 0;
-    
-    my $convert_to = $self->{'convert_to'};
-    my $paged_txt_output_mode = ($convert_to =~ /(pagedimgtxt|paged_text)/) ? 1 : 0;
-    
-    # the following line is necessary to avoid 'uninitialised variable' error
-    # messages concerning the converted_to member variable when PDFPlugin's 
-    # use_sections option is checked.
-    # PDFBox plugin now processes use_sections option, when working with v1.5.0
-    # of the PDFBox jar file (which embeds each page in special <div> tags).
-    if ($target_file_type eq "html") {
-    $self->{'converted_to'} = "HTML";
-    } elsif ($target_file_type eq "jpg" || $target_file_type eq "png") { # || $target_file_type eq "gif"
-    # GIF not supported by PDFBox at present, see https://pdfbox.apache.org/1.8/commandline.html#pdftoimage
-    $self->{'converted_to'} = $target_file_type;    
-    $img_output_mode = 1;
-    } else {
-    $self->{'converted_to'} = "text";
-    }
-
-    my $outhandle = $self->{'outhandle'};
-    my $verbosity = $self->{'verbosity'};
-
-    my $source_file_no_path = &File::Basename::basename($source_file_full_path);
-    # Determine the full name and path of the output file
-    my $target_file_path;
-    if ($self->{'enable_cache'}) {
-    $self->init_cache_for_file($source_file_full_path);
-    my $cache_dir = $self->{'cached_dir'};
-    my $file_root = $self->{'cached_file_root'};
-    #$file_root .= "_$convert_id" if ($convert_id ne "");
-
-    # append the output filetype suffix only for non-image output formats, since for
-    # images we can be outputting multiple image files per single PDF input file
-    my $target_file = ($img_output_mode || $paged_txt_output_mode) ? "$file_root" : "$file_root.$target_file_type";
-
-    $target_file_path = &FileUtils::filenameConcatenate($cache_dir,$target_file);
+sub checker {
+    my $self = shift(@_);
+    my ($doc_obj, $source_file_full_path) = @_;
+
+    print STDERR "***** @@@@@@ !!!!!!! MalwareCheckerConvert::convert called!\n";
+
+
+    print STDERR "**** want to check: $source_file_full_path\n";
+
+    my $virustotal_apikey = $self->{'virustotal_apikey'};
+  
+    # Build up command along the lines:
+    #   java -cp $ENV{'GEXT_MALWARECHECK'}/lib/java/malware-checker-1.0-SNAPSHOT-jar-with-dependencies.jar org.greenstone.virustotal.ScanFile
+
+    my $jar_file = &FileUtils::filenameConcatenate($ENV{'GEXT_MALWARECHECKER'},"lib","java","malware-checker-1.0-SNAPSHOT-jar-with-dependencies.jar");
+        
+    my $checker_cmd = "java -cp $jar_file org.greenstone.virustotal.ScanFile $virustotal_apikey $source_file_full_path";
+    
+    my $print_info = { 'message_prefix' => "MalwareChecker Conversion",
+               'message' => "Checking $source_file_full_path" };
+    # $print_info->{'cache_mode'} = $cache_mode if ($cache_mode ne "");
+
+    #my $target_file_path = ".virustotal-resourceid-" + $source_file_full_path;
+    my $target_file_path = $source_file_full_path . "-VIRUSTOTAL-RESOURCEID";
+    
+    my ($regenerated,$result,$had_error) 
+    = $self->autorun_general_cmd($checker_cmd,$source_file_full_path,$target_file_path,$print_info);
+
+    if ($had_error) {
+    print STDERR "Error: Failed to run cmd: $checker_cmd\n";
     }
     else {
-    # this is in gsdl/tmp. get a tmp filename in collection instead???
-    $target_file_path = &util::get_tmp_filename($target_file_type);
-
-    # for image files, remove the suffix, since we can have many output image files
-    # per input PDF (one img for each page of the PDF, for example)
-    if($img_output_mode || $paged_txt_output_mode) {
-        $target_file_path =~ s/\.[^.]*$//g;
-        if(!&FileUtils::directoryExists($target_file_path)) {       
-        mkdir($target_file_path);
+    chomp($result);
+    my $top_section = $doc_obj->get_top_section();
+    $doc_obj->add_utf8_metadata($top_section, "VirusTotalResourceID", $result);
+    }
+
+    # Need to regulate how often we make calls to VirusTotal API
+    # If public key, then 4 per min
+
+    if ($self->{'virustotal_keytype'} eq "public") {
+### if ($regenerated) {
+        my $verbosity = $self->{'verbosity'};
+        if ($verbosity >= 1) {
+        my $outhandle = $self->{'outhandle'};
+        print $outhandle "Sleeping for 15 secs, in accordance with public key VirusTotal API Terms and Conditions\n";
         }
-        
-        # once the item file for the imgs has been created, need to adjust target_file_path
-
-        # below, we'll store the dir just created to pbtmp_file_paths, so all imgs and the
-        # item file generated in it can be deleted in one go on clean_up
-    }
-    
-    push(@{$self->{'pbtmp_file_paths'}}, $target_file_path);
-    }
-
-    # Generate and run the convert command
-    my $convert_cmd = "";
-
-    # want the filename without extension, because any images 
-    # are to be generated with the same filename as the PDF
-    my ($tailname, $dirname, $suffix) = &File::Basename::fileparse($source_file_full_path, "\\.[^\\.]+\$");
-
-    if($img_output_mode || $paged_txt_output_mode) { # converting each page to image and/or text
-    my $output_prefix = &FileUtils::filenameConcatenate($target_file_path, $tailname);
-
-    # Our custom class does renaming of the pages (simplified to just numbers) for PagedImagePlugin
-    #$convert_cmd = $paged_txt_output_mode ? $self->{'malwarechecker_imgtxt_launch_cmd'} : $self->{'malwarechecker_img_launch_cmd'};
-    $convert_cmd = $self->{'malwarechecker_imgtxt_launch_cmd'};
-    $convert_cmd .= " -imagesOnly" unless($paged_txt_output_mode); # set to images only unless there's text too
-    if($img_output_mode) { # whether images-only or images-and-text mode
-        $convert_cmd .= " -imageType $target_file_type";
-        $convert_cmd .= " -dpi ". $self->{"dpi"} if defined $self->{"dpi"};
-    } else { # img_output_mode off, so paged txt only and no images
-        $convert_cmd .= " -textOnly";       
-    }
-    $convert_cmd .= " -outputPrefix \"$output_prefix\"";
-    $convert_cmd .= " \"$source_file_full_path\"";
-    
-    } else { # single stream of text or html
-    
-    if ($target_file_type eq "html") {
-        $convert_cmd = $self->{'malwarechecker_html_launch_cmd'};
-        $convert_cmd .= " -html" if ($target_file_type eq "html");
-    } else {
-        $convert_cmd = $self->{'malwarechecker_txt_launch_cmd'};
-    }
-    $convert_cmd .= " \"$source_file_full_path\" \"$target_file_path\"";
-    }
-
-    if ($verbosity>2) {
-    &gsprintf($outhandle,"Convert command: $convert_cmd\n");
-    }
-
-    my $print_info = { 'message_prefix' => "PDFBox Conversion",
-               'message' => "Converting $source_file_no_path to: $target_file_type" };
-    # $print_info->{'cache_mode'} = $cache_mode if ($cache_mode ne "");
-
-    my ($regenerated,$result,$had_error) 
-    = $self->autorun_general_cmd($convert_cmd,$source_file_full_path, $target_file_path,$print_info);
-
-    if($img_output_mode || $paged_txt_output_mode) {
-    # now the images have been generated, generate the "$target_file_path/tailname.item" 
-    # item file for them, which is also the target_file_path that needs to be returned
-    $target_file_path = &util::create_itemfile($target_file_path, $tailname, $target_file_type);
-    #print STDERR "**** item file: $target_file_path\n";
-    }
-    elsif ($self->{'converted_to'} eq "text") {
-    # ensure html entities are doubly escaped for pdfbox to text conversion: &amp; -> &amp;amp;
-    # conversion to html does it automatically, but conversion to text doesn't
-    # and this results in illegal characters in doc.xml
-
-    my $fulltext = &FileUtils::readUTF8File($target_file_path);
-    if(defined $fulltext) {
-        #$fulltext = &HTML::Entities::encode($fulltext); # doesn't seem to help
-        $fulltext =~ s@&@&amp;@sg; # Kathy's fix to ensure doc contents don't break XML
-        &FileUtils::writeUTF8File($target_file_path, \$fulltext);
-    } else {
-        print STDERR "PDFBoxConverter::convert(): Unable to read from converted file\n";
-        $had_error = 1;
-    }
-    }
-
-    if ($had_error) {
-    return (0, $result,$target_file_path);
-    }
-    return (1, $result,$target_file_path);
-}
-
-sub convert_without_result {
-    my $self = shift(@_);
-
-    my $source_file_path = shift(@_);
-    my $target_file_type = shift(@_);
-    my $convert_options  = shift(@_) || "";
-    my $convert_id       = shift(@_) || "";
-
-    return $self->convert($source_file_path,$target_file_type,
-              $convert_options,$convert_id,"without_result");
+        sleep(15);
+### }
+    }       
 }
 

gs2-extensions/malware-checker/trunk/perllib/plugins/PDFv3Plugin.pm

@@ -112 +112 @@
      ];
 
+my $virustotal_keytype_list =
+    [ { 'name' => "public",
+    'desc' => "{PDFv3Plugin.virustotal_type_list.public}" },
+      { 'name' => "private",
+    'desc' => "{PDFv3Plugin.virustotal_type_list.private}" }
+    ];
+
+      
 my $opt_malwarechecker_args = [ { 'name' => "malware_checker",
                   'desc' => "{MalwareChecker.malware_checker}",
                   'type' => "flag",               
-                  'reqd' => "no" } ];
-
-
-my $options = { 'name'     => "PDFv2Plugin",
+                  'reqd' => "no" },
+                { 'name' => "virustotal_keytype",
+                  'desc' => "{PDFv3Plugin.virustotal_keytype}",
+                  'type' => "enum",
+                  'list' => $virustotal_keytype_list,                 
+                  'deft' => "public" },
+                { 'name' => "virustotal_apikey",
+                  'desc' => "{PDFv3Plugin.virustotal_apikey}",
+                  'type' => "string",
+                  'deft' => "CHANGEME" }
+    ];
+
+
+my $options = { 'name'     => "PDFv3Plugin",
         'desc'     => "{PDFv2Plugin.desc}",
         'abstract' => "no",
@@ -756 +774 @@
     print STDERR "\n\n\n\n\n***** PDFv3Plugin::process()\n";
     if ($self->{'malware_checker'}) {
+
     my $filename = &FileUtils::filenameConcatenate($base_dir, $file);
-    print STDERR "**** want to check: $filename\n";
-
-    # java -cp $ENV{'GEXT_MALWARECHECK'}/lib/java/malware-checker-1.0-SNAPSHOT-jar-with-dependencies.jar org.greenstone.virustotal.ScanFile
-
-    my $cmd = "java -cp $ENV{'GEXT_MALWARECHECKER'}/lib/java/malware-checker-1.0-SNAPSHOT-jar-with-dependencies.jar org.greenstone.virustotal.ScanFile $filename";
-    my $status = system($cmd);
-    print "STATUS = $status\n";
+
+    $self->MalwareCheckerConverter::checker($doc_obj,$filename);
     }