Changeset 34778

Timestamp:

2021-02-01T17:03:58+13:00 (3 years ago)

Author:

davidb

Message:

Next logical commit point after testing on CSV files in amc-essentia/import

Location:

main/trunk/model-sites-dev/mars

Files:

• RUN-APPLY-AROUSAL-MODEL.sh (modified) (2 diffs)
• WekaApplyArousalModel.java (modified) (3 diffs)
• WekaTrainArousalModel.java (modified) (2 diffs)

main/trunk/model-sites-dev/mars/RUN-APPLY-AROUSAL-MODEL.sh

@@ -1 +1 @@
 #!/bin/bash
+
+input_data_filename=${1:-collect/deam/etc/deam-essentia-features-arousal-valence.arff}
+output_data_filename=${2:-predicted-arousal.csv}
 
 . ./_set_weka_classpath.bash
@@ -5 +8 @@
 java -cp "$cp_args" WekaApplyArousalModel \
      "reptree-arousal-serialized.model" \
-     "collect/deam/etc/deam-essentia-features-arousal-valence.arff" \
-     "predicted-arousal.csv"
+     "$input_data_filename" \
+     "$output_data_filename" 
+
+echo "****"
+echo "* Saved output data: $output_data_filename"
+echo "****"
+
 
 

main/trunk/model-sites-dev/mars/WekaApplyArousalModel.java

@@ -10 +10 @@
 
 import weka.core.converters.ConverterUtils.DataSource;
+import weka.core.Attribute;
 import weka.core.Instance;
 import weka.core.Instances;
 import weka.core.SerializationHelper;
 
-//import weka.filters.Filter;
-//import weka.filters.unsupervised.attribute.Remove;
-
-//import weka.classifiers.Evaluation;
+import weka.filters.Filter;
+import weka.filters.unsupervised.attribute.Remove;
+
 import weka.classifiers.Classifier;
-//import weka.classifiers.trees.REPTree;
-
-
-
-//import weka.core.Instances;
 
 
@@ -31 +26 @@
 class WekaApplyArousalModel
 {
+    public final static String AROUSAL_ATTRIBUTE_NAME = "arousal_sample_26500ms";
+    public final static String VALANCE_ATTRIBUTE_NAME = "valance_sample_26500ms";
+
+    public final static String PREDICT_ATTRIBUTE_NAME = "arousal_sample_26500ms";
+    
+
+    public static Instances applyFilter(Instances data_instances,String additional_remove)
+    {
+    String remove_option_args = ("-R 425,458-466"); // remove ordinal attributes
+    if (additional_remove != null) {
+        // top up, e.g. '472' representing valance in ground-truth files
+        remove_option_args += "," + additional_remove;
+    }
+
+
+    Instances filtered_data_instances = null;
+    try {
+        String[] filter_options = weka.core.Utils.splitOptions(remove_option_args); // remove ordinal attributes and 'valance'
+        Remove filter_remove = new Remove();
+        filter_remove.setOptions(filter_options);
+        filter_remove.setInputFormat(data_instances);
+        filtered_data_instances = Filter.useFilter(data_instances, filter_remove);
+    }
+    catch (Exception e) {
+        e.printStackTrace();
+        System.exit(1);
+    }
+    
+    return filtered_data_instances; 
+    }
+
+
+    
+    public static void checkDatasetInstancesCompatible(Instances new_instances)
+    {
+
+    final String gt_datasource_filename = "collect/deam/etc/deam-essentia-features-arousal-valence.arff";
+
+    try {
+        DataSource gt_data_source = new DataSource(gt_datasource_filename);
+        Instances gt_instances = gt_data_source.getDataSet();
+        gt_instances = applyFilter(gt_instances,"472"); // top up with removing 'valance'
+        
+        gt_instances.setClassIndex(gt_instances.numAttributes() - 1);
+
+        String equal_header_message = gt_instances.equalHeadersMsg(new_instances);
+        
+        if (equal_header_message != null) {
+        System.err.println("Headers to datasets were not equal!");
+        System.err.println(equal_header_message);
+        System.exit(1);
+        }
+    }
+    catch (Exception e) {
+        e.printStackTrace();
+        System.exit(1);
+    }
+
+    }
+    
     public static void main(String[] args)
     {
@@ -49 +104 @@
         BufferedInputStream bis= new BufferedInputStream(fis);
         Classifier classifier = (Classifier)SerializationHelper.read(bis);
-    
-    
-
-        // load unlabeled data
+
+        //
+        // Load in unlabeled data
+        //
+        
         System.out.println("Loading unlabeled instances: " + unclassified_data_input_filename);
-        FileReader fr = new FileReader(unclassified_data_input_filename);
-        BufferedReader br = new BufferedReader(fr);
-        Instances gt_instances = new Instances(br);
-        br.close();
-
-        // set class attribute
-        gt_instances.setClassIndex(gt_instances.numAttributes() - 2);
-        
-        // create copy
-        Instances unlabeled_instances = new Instances(gt_instances);    
+
+        //FileReader fr = new FileReader(unclassified_data_input_filename);
+        //BufferedReader br = new BufferedReader(fr);
+        //Instances unlabeled_instances = new Instances(br);
+        //br.close();
+
+        DataSource data_source = new DataSource(unclassified_data_input_filename);
+        Instances unlabeled_instances = data_source.getDataSet();
+
+        // Work out if we're dealing with a ground-truth ARFF file or not
+        // (i.e. already has the desired attribute)
+
+        Instances groundtruth_instances = null;     
+        Attribute predict_attribute = unlabeled_instances.attribute(PREDICT_ATTRIBUTE_NAME);
+        
+        if (predict_attribute == null) {
+
+        unlabeled_instances = applyFilter(unlabeled_instances,null); // no additional top-up to remove      
+        int num_attributes = unlabeled_instances.numAttributes();
+
+        Attribute arousal_attribute = new Attribute(AROUSAL_ATTRIBUTE_NAME);
+        unlabeled_instances.insertAttributeAt(arousal_attribute,num_attributes);
+        num_attributes++;
+        
+        //Attribute valance_attribute = new Attribute(VALANCE_ATTRIBUTE_NAME);
+        //unlabeled_instances.insertAttributeAt(valance_attribute,num_attributes);
+        //num_attributes++;
+
+        //predict_attribute = new Attribute(PREDICT_ATTRIBUTE_NAME);
+        //unlabeled_instances.insertAttributeAt(predict_attribute,num_attributes);
+        //unlabeled_instances.setClassIndex(num_attributes);
+        //num_attributes++;
+
+        //predict_attribute = unlabeled_instances.attribute(PREDICT_ATTRIBUTE_NAME);
+        //unlabeled_instances.setClass(predict_attribute);
+
+        //unlabeled_instances.setClassIndex(num_attributes - 1);
+        }
+        else {
+        // Dealing with ground-truth data:
+        //   => already has 'arousal' attribute
+        //   => in fact has 'valance' attribute too, which we want to remove
+        
+        unlabeled_instances = applyFilter(unlabeled_instances,"472"); // top-up with removal of 'valance'
+        //unlabeled_instances.setClass(predict_attribute);
+        
+        // reference share this as 'groundtruth_instances' to trigger error calculation and output
+        groundtruth_instances = unlabeled_instances;
+        }
+
+        int num_attributes = unlabeled_instances.numAttributes();
+        unlabeled_instances.setClassIndex(num_attributes - 1);
+        
+        checkDatasetInstancesCompatible(unlabeled_instances);
+        
+        // Set class attribute
+        //unlabeled_instances.setClassIndex(gt_instances.numAttributes() - 2);
+
+
+        // Create copy where the predictions are to be made
         Instances labeled_instances = new Instances(unlabeled_instances);
-    
-        // label instances
+
+        //
+        // Label instances
+        //
+        
         final int num_instances = unlabeled_instances.numInstances();       
         for (int i=0; i<num_instances; i++) {
         Instance unlabeled_instance = unlabeled_instances.instance(i);
+
         System.out.print("Making prediction for: " + i + "/" + num_instances);
-        double classified_label = classifier.classifyInstance(unlabeled_instance);
-        labeled_instances.instance(i).setClassValue(classified_label);
-
-        Instance gt_instance = gt_instances.instance(i);
-        double gt_class_value = gt_instance.classValue();
-        System.out.println("  error: " + Math.abs(classified_label-gt_class_value));
-        }
-
-
-        // save labeled data            
+        double classified_value = classifier.classifyInstance(unlabeled_instance);
+        labeled_instances.instance(i).setClassValue(classified_value);
+
+        String formatted_classified_value = String.format("% 06.3f", classified_value);
+        
+        System.out.print(" value = " + formatted_classified_value);
+        
+        if (groundtruth_instances != null) {
+            Instance gt_instance = groundtruth_instances.instance(i);
+            double gt_class_value = gt_instance.classValue();
+            double error = Math.abs(classified_value - gt_class_value);
+
+            String formatted_error = String.format("%.3f", error);
+            System.out.print("  [error: " + formatted_error + "]");
+        }
+        System.out.println();
+        }
+
+        //
+        // Save labeled data
+        //
+        
         System.out.println("Saving labeled instances: " + classified_data_output_filename);     
         FileWriter fw = new FileWriter(classified_data_output_filename);

main/trunk/model-sites-dev/mars/WekaTrainArousalModel.java

@@ -36 +36 @@
     if (args.length != 2) {
         System.err.println("Error: incorrect number of command-line arguments");
-        System.err.println("Usage: input_training_data.arff output-model.ser");
+        System.err.println("Usage: input_training_data.arff output-model.{model|ser}");
         System.exit(1);
     }
@@ -51 +51 @@
         //String[] filter_options = weka.core.Utils.splitOptions("-R 472");
         //String[] filter_options = weka.core.Utils.splitOptions("-R 472 -R 458-466");
-        String[] filter_options = weka.core.Utils.splitOptions("-R 472,458-466");
+        String[] filter_options = weka.core.Utils.splitOptions("-R 425,458-466,472"); // remove ordinal attributes and 'valance'
         Remove filter_remove = new Remove();
         filter_remove.setOptions(filter_options);