Changeset 36859

Timestamp:

2022-10-22T15:41:14+13:00 (18 months ago)

Author:

davidb

Message:

Coding developments that mean param passed arousal and valence values not used; query_resutls_ capped to max_docs_

Location:

gs3-extensions/mars-src/trunk/src/java/org/greenstone/gsdl3

Files:

• service/GS2WekaDBSearch.java (modified) (4 diffs)
• util/WekaDBWrapper.java (modified) (3 diffs)
• util/WekaFindInstanceKNN.java (modified) (2 diffs)

gs3-extensions/mars-src/trunk/src/java/org/greenstone/gsdl3/service/GS2WekaDBSearch.java

@@ -51 +51 @@
     protected static final String RADIUS_PARAM  = "radius";
     protected static final String MAXDOCS_PARAM = "maxDocs";
+    protected static final String AROUSAL_PARAM  = "arousal";
+    protected static final String VALENCE_PARAM  = "valence";
 
     //protected static final String WEKA_MODEL_DEFAULT_DIRECTORY = "weka-model";
@@ -115 +117 @@
         setStandardQueryParams (params); // ****
         
-        this.wekadb_src.runQuery (weka_db_index_dir, KNN_FEATURES_FILENAME, assoc_index_dir, query);
-        Vector docs = this.wekadb_src.getQueryResult ();
+        this.wekadb_src.runQuery(weka_db_index_dir, KNN_FEATURES_FILENAME, assoc_index_dir, query);
+        Vector docs = this.wekadb_src.getQueryResult();
 
         if (docs.isEmpty()) {
@@ -145 +147 @@
         result.appendChild (document_list);
         for (int d = 0; d < docs.size (); d++) {
-            WekaDBDocInfo adb_doc = (WekaDBDocInfo) docs.elementAt(d);
-
-            String doc_id  = adb_doc.getDocID();
-            double rank    = adb_doc.getTopRank();
-            String offsets = adb_doc.getOffsetList();
+            WekaDBDocInfo wdb_doc = (WekaDBDocInfo) docs.elementAt(d);
+
+            String doc_id  = wdb_doc.getDocID();
+            double rank    = wdb_doc.getTopRank();
+            String offsets = wdb_doc.getOffsetList();
 
             Element doc_node = createDocNode (result_doc, doc_id, Double.toString (rank));
@@ -191 +193 @@
         int docs = Integer.parseInt(value);
         this.wekadb_src.setMaxDocs(docs);
+        }
+        else if (name.equals(AROUSAL_PARAM)) {
+        double arousal = Double.parseDouble(value);
+        this.wekadb_src.setArousal(arousal);
+        }       
+        else if (name.equals(VALENCE_PARAM)) {
+        double valence = Double.parseDouble(value);
+        this.wekadb_src.setValence(valence);        
         } // ignore any others
     }

gs3-extensions/mars-src/trunk/src/java/org/greenstone/gsdl3/util/WekaDBWrapper.java

@@ -45 +45 @@
 
     protected int offset_ = 100;
-    protected int length_ = 20;
+    protected int length_ = 20; // **** Unused
     
     // Approximate matching not yet utilized
-    protected double radius_;
+    protected double radius_;   // **** Unused
 
     protected int max_docs_;
 
+    protected double arousal_;
+    protected double valence_;
+    
     static Logger logger = Logger.getLogger (org.greenstone.gsdl3.util.WekaDBWrapper.class.getName ());
 
@@ -76 +79 @@
     radius_ = radius;
     }
-
+    
     public void setMaxDocs(int max_docs) {
     max_docs_ = max_docs;
     }
 
+    public void setArousal(double arousal) {
+    arousal_ = arousal;
+    }
+    public void setValence(double valence) {
+    valence_ = valence;
+    }
+    
     /** returns a string with all the current query param settings */
     // the following was in MG version, do we need this in WekaDB version? // ****
@@ -142 +152 @@
     WekaFindInstanceKNN.init(full_knn_model_filename);
 
-    //Instances nearest_instances = WekaFindInstanceKNN.kNearestNeighbours("ds_22716_5743",6,-0.549489,-0.118439,3);
-    String doc_id = query_string;
-
-    int segment = 6;
-    //String segment_str = Integer.toString(segment);
-    String doc_id_segment =  doc_id +"-" + segment;
-
-    double arousal_val = -0.549489;
-    double valence_val =  -0.118439;
-    int k_nearest_num = 3;
-    
-    System.err.println("doc_id_segment = " + doc_id_segment);
-
-    
-    Instances nearest_instances = WekaFindInstanceKNN.kNearestNeighbours(doc_id_segment,arousal_val,valence_val,k_nearest_num);
-
-        
-    query_result_ = new Vector();
-
-    /*
-    WekaDBDocInfo wekaDB_doc_info1 = new WekaDBDocInfo("ds_20415_2337",0.9,4);
-    query_result_.add(wekaDB_doc_info1);
-
-    WekaDBDocInfo wekaDB_doc_info2 = new WekaDBDocInfo("ds_51017_15513",0.87,1);
-    query_result_.add(wekaDB_doc_info2);
-
-    WekaDBDocInfo wekaDB_doc_info3 = new WekaDBDocInfo("ds_20415_2337",0.82,6);
-    query_result_.add(wekaDB_doc_info3);
-    */
-
-    int clamped_k_nearest_num = Math.min(k_nearest_num,nearest_instances.size());
-    
-    for (int i=0; i<clamped_k_nearest_num; i++) {
-        Instance instance = nearest_instances.instance(i);
+    String doc_id  = query_string;
+    int    segment = offset_;
+
+    String query_doc_id_segment =  doc_id + "-" + segment;
+
+    double query_arousal_val = arousal_;
+    double query_valence_val =  valence_;
+
+    int k_nearest_num = max_docs_;
+    int expanded_k_nearest_num = max_docs_ * 5; // * internally get more matches, then sift through to arrive at the best 'max_docs_'
+    
+    Pattern doc_seg_re = Pattern.compile("^(\\w+)-(\\d+)$");
+    //Matcher query_doc_seg_match = doc_seq_re.matcher(query_doc_id_segment);
+    
+    Instances nearest_instances
+        = WekaFindInstanceKNN.kNearestNeighbours(query_doc_id_segment,query_arousal_val,query_valence_val,k_nearest_num);
+
+        
+    Vector expanded_query_result = new Vector();
+
+    int nearest_instances_len = nearest_instances.size();
+    
+    int clamped_expanded_k_nearest_num = Math.min(expanded_k_nearest_num,nearest_instances_len);
+
+    double pos_penalty = 0.1;
+    
+    for (int ei=0; ei<clamped_expanded_k_nearest_num; ei++) {
+        Instance instance = nearest_instances.instance(ei);
         logger.info("\tProcessing returned instance: " + instance);
         
-        //Attribute doc_id_segment_att = instance.attribute(0);
-        //String matching_doc_id_segment = instance.attribute(0).value(0);
         String matching_doc_id_segment = instance.stringValue(0);
-        //double arousal_val    = instance.attribute(1);
-        //double valence_val    = instance.attribute(2);
-
-
-        Pattern p = Pattern.compile("^(\\w+)-(\\d+)$");
-        Matcher m = p.matcher(matching_doc_id_segment);
+
+        
+        //Pattern p = Pattern.compile("^(\\w+)-(\\d+)$");
+        Matcher m = doc_seg_re.matcher(matching_doc_id_segment);
         if (m.matches()) {
 
         String matching_doc_id = m.group(1);
         int matching_segment_offset = Integer.parseInt(m.group(2));
-        
-        double matching_rank = 0.9;
+
+        if (matching_doc_id.equals(doc_id)) {
+            continue;
+        }
+        
+        double matching_arousal_val = instance.value(1);
+        double matching_valence_val = instance.value(2);
+        
+        double matching_diff = (Math.abs(query_arousal_val - matching_arousal_val)
+                    + Math.abs(query_valence_val - matching_valence_val))/4.0;
+        double matching_rank = 1.0 - matching_diff - (pos_penalty * (double)ei);
 
         logger.info("\tAdding in: matching_doc_id = " + matching_doc_id);
         WekaDBDocInfo wekaDB_doc_info = new WekaDBDocInfo(matching_doc_id,matching_rank,matching_segment_offset);
-        query_result_.add(wekaDB_doc_info);
+        expanded_query_result.add(wekaDB_doc_info);
         }
         else {
-        logger.error("Returned AV k-nearest neighbour match '"+doc_id_segment+"' could not be parsed as <doc-id>-<segment>" );
+        logger.error("Returned AV k-nearest neighbour match '"+matching_doc_id_segment+"' could not be parsed as <doc-id>-<segment>" );
+        }          
+    }
+    
+    query_result_ = new Vector();
+
+    int i = 0;
+    while (i < k_nearest_num) {
+        if (i >= expanded_query_result.size()) {
+        break;
         }
-    }
-    
-    /*
-    int num_matches_within_track = 6;
-
-
-    first_entry = addQueryResult(first_entry,root_doc_id,rankVector,offsetVector);
-    // and now reset vectors to empty to be ready for next chain of values
-    rankVector = new Vector<Double>();
-    offsetVector = new Vector<Integer>();
-
-
-    rankVector.add(rank);
-    offsetVector.add(target_frame);
-    
-    addQueryResult(first_entry,root_doc_id,rankVector,offsetVector);
-    */
-
+        
+        query_result_.add(expanded_query_result.get(i));
+        i++;
+    }
+
+    Collections.sort(query_result_);    
     }
     

gs3-extensions/mars-src/trunk/src/java/org/greenstone/gsdl3/util/WekaFindInstanceKNN.java

@@ -93 +93 @@
     public static Instances kNearestNeighbours(String doc_id_segment, double arousal_val,double valence_val, int k_nearest_num)
     {
+    System.err.println("**** wekaFindInstnaceKNN::kNearestNeighbours() called with:");
+    System.err.print(  "**** doc_id_segment = " + doc_id_segment);
+    System.err.print(  " arousal_val = "   + arousal_val);
+    System.err.print(  " valence_val = "   + valence_val);
+    System.err.println(" k_nearest_num = " + k_nearest_num);
     
     Instance sample_instance = new DenseInstance(3);
@@ -103 +108 @@
     //sample_instance.setValue(2, -0.118439);
 
-    //String segment_str = Integer.toString(segment);
-    //sample_instance.setValue(0, doc_id +"-" + segment_str);
 
     sample_instance.setValue(0, doc_id_segment);
-
+    
     sample_instance.setValue(1, arousal_val);
     sample_instance.setValue(2, valence_val);