Changeset 35196

Timestamp:

2021-05-30T23:12:01+12:00 (3 years ago)

Author:

davidb

Message:

av.json files now read in and appended to central CSV file

Location:

gs3-extensions/mars-src/trunk/perllib

Files:

• wekaDBBuilder.pm (modified) (1 diff)
• wekaDBBuildproc.pm (modified) (6 diffs)

gs3-extensions/mars-src/trunk/perllib/wekaDBBuilder.pm

@@ -78 +78 @@
     # init Weka CSV feature file
     
-    my $adb_filename = &util::filename_cat($wekaDB_dir,"lsh-features.adb");
+    my $weka_features_filename = &util::filename_cat($wekaDB_dir,"av-features.csv");
 
-    print $outhandle "\n    creating Weka CSV Feature File\n"  if ($verbosity >= 1);
+    print $outhandle "\n    creating Weka CSV Features File\n"  if ($verbosity >= 1);
 
-    my $init_cmd = "audioDB -N -d $adb_filename";
-    my $init_status = system($init_cmd);
-    if ($init_status != 0) {
-        print STDERR "Error: failed to initialize the audioDB database\n";
-        print STDERR "         $adb_filename\n";
+    # Create a CSV features file with the headline in it
+    if (open(my $WEKA_FOUT, '>', $weka_features_filename)) {
+        binmode($WEKA_FOUT,":utf8");
+        print $WEKA_FOUT "IdentifierWithTimeOffset,Arousal,Valence\n";
+        close($WEKA_FOUT);
+    }
+    else {
+        print STDERR "Error: failed to create:\n";
+        print STDERR "         $weka_features_filename\n";
         print STDERR "       $!\n";
-        if ($verbosity>=2) {
-        print STDERR "       cmd: $init_cmd\n";
-        }
         return;
     }
-
+    
     }
 

gs3-extensions/mars-src/trunk/perllib/wekaDBBuildproc.pm

@@ -1 @@
-##########################################################################
+#########################################################################
 #
-# wekaDBBuildproc.pm -- 
+# wekaDBBuildproc.pm -- builds up a CSV file to be used by Weka
 # A component of the Greenstone digital library software
 # from the New Zealand Digital Library Project at the 
@@ -24 +24 @@
 ###########################################################################
 
-# This document processor outputs a document for indexing (should be 
-# implemented by subclass) and storing in the database
-
 package wekaDBBuildproc;
 
@@ -32 +29 @@
 no strict 'refs'; # allow filehandles to be variables and viceversa
 
+use multiread;
+use Encode;
+use JSON;
+
 use util;
 
 use extrabuildproc;
 
+my $av_time_slice    = 6; # 6 secs
+my $av_frame_overlap = 3; # 3 secs
 
 BEGIN {
@@ -50 +53 @@
 }
 
+
+sub read_json_file
+{
+    my $self = shift (@_);
+    my ($json_filename) = @_;
+
+    my $json_decoded_data = undef;
+    
+    if (open(JSON_FILE, "$json_filename")) {
+    my $json_file_content;
+    
+    my $json_file_reader = new multiread();
+    $json_file_reader->set_handle('wekaDBBuildproc::JSON_FILE');
+    $json_file_reader->read_file(\$json_file_content);
+
+    # Next 2 lines result in an UTF8-friendly string
+    my $json_file_content_bytes = encode('UTF-8', $json_file_content);
+    $json_file_content = decode("utf8",$json_file_content_bytes);
+    
+    $json_decoded_data = JSON->new->utf8->decode($json_file_content);
+    
+    close(JSON_FILE);
+    }
+    else {
+    print STDERR "wekaDBBuildproc::read_json_file(): Failed to open $json_filename\n";
+    print STDERR "  $!\n";
+
+    }
+
+    
+    return $json_decoded_data;
+}
 
 sub textedit {
@@ -66 +101 @@
     my $build_dir  = $self->{'build_dir'};
 
-    # full path to adb database
-    my $adb_filename 
-    = &util::filename_cat($build_dir, "wekaDB", "lsh-features.adb");
+    # full path to Weka CSV file
+    my $weka_features_filename 
+    = &util::filename_cat($build_dir, "wekaDB", "av-features.csv");
 
     # get doc id
@@ -75 +110 @@
     # map to assoc dir
     my $top_section = $doc_obj->get_top_section();
-    my $assoc_file
-    = $doc_obj->get_metadata_element ($top_section,"assocfilepath");
-    my $assoc_filename = &util::filename_cat($source_dir,$assoc_file);
 
-    my $chr12_filename = &util::filename_cat($assoc_filename,"doc.chr12");
-    my $powerlog_filename = &util::filename_cat($assoc_filename,"doc.power");
+    my $assoc_file       = $doc_obj->get_metadata_element ($top_section,"assocfilepath");
+    my $assoc_filename   = &util::filename_cat($source_dir,$assoc_file);
 
-    print $outhandle "  Inserting features for $doc_oid\n";
+    my $av_json_filename = &util::filename_cat($assoc_filename,"av.json");
+    my $av_json_hashmap  = $self->read_json_file($av_json_filename);
 
-    my $cmd = "audioDB -d \"$adb_filename\" -I -k \"$doc_oid\" -f \"$chr12_filename\" -w \"$powerlog_filename\"";
+    if (!defined $av_json_hashmap) {
+    return;
+    }
+    
+    print $outhandle "  Appending features for $doc_oid\n";
 
-    my $status = system($cmd);
-    if ($status != 0) {
-    print STDERR "Error: failed to run:\n  $cmd\n$!\n";
+    my $AV_OUT;
+    if (!open($AV_OUT,">>","$weka_features_filename")) {
+    print STDERR "Failed to append to $weka_features_filename\n";
+    print STDERR "  $!\n";
+    return;
     }
+    
+    binmode($AV_OUT,":utf8");
+
+    my $arousal_vals = $av_json_hashmap->{"arousal"};
+    my $valence_vals = $av_json_hashmap->{"valence"};
+
+    my $num_vals = scalar(@{$arousal_vals});
+
+    my $t = $av_time_slice;
+    for (my $i=0; $i<$num_vals; $i++) {
+    my $arousal_val = $arousal_vals->[$i];
+    my $valence_val = $valence_vals->[$i];
+
+    print $AV_OUT "$doc_oid-$t,$arousal_val,$valence_val\n";
+    $t += $av_frame_overlap;
+    }
+
+    close($AV_OUT);
 }