Changeset 36857 for gs3-extensions

Timestamp:

2022-10-22T10:09:15+13:00 (18 months ago)

Author:

davidb

Message:

Service now showing some basic functionality to retrieval songs

Location:

gs3-extensions/mars-src/trunk/src/java/org/greenstone/gsdl3

Files:

• service/GS2WekaDBSearch.java (moved) (moved from gs3-extensions/mars-src/trunk/src/java/org/greenstone/gsdl3/service/GS2WekaAVRecommender.java ) (5 diffs)
• util/WekaDBWrapper.java (modified) (3 diffs)

gs3-extensions/mars-src/trunk/src/java/org/greenstone/gsdl3/service/GS2WekaDBSearch.java

@@ -1 +1 @@
 /*
- *    GS2WekaAVRecommender.java
+ *    GS2WekaDBSearch.java
  *    Copyright (C) 2011 New Zealand Digital Library, http://www.nzdl.org
  *
@@ -43 +43 @@
 // 'Search'
 
-public class GS2WekaAVRecommender extends AbstractGS2AudioSearch {
-
-    static Logger logger = Logger.getLogger (org.greenstone.gsdl3.service.GS2WekaAVRecommender.class.getName ());
+public class GS2WekaDBSearch extends AbstractGS2AudioSearch {
+
+    static Logger logger = Logger.getLogger (org.greenstone.gsdl3.service.GS2WekaDBSearch.class.getName ());
 
     protected static final String OFFSET_PARAM  = "offset";
@@ -52 +52 @@
     protected static final String MAXDOCS_PARAM = "maxDocs";
 
-    protected static final String WEKA_MODEL_DEFAULT_DIRECTORY = "weka-model";
-    protected static final String KNN_MODEL_FILENAME = "av-and-features-knn.ser";
+    //protected static final String WEKA_MODEL_DEFAULT_DIRECTORY = "weka-model";
+    //protected static final String KNN_MODEL_FILENAME = "av-and-features-knn.ser";
+
+    protected static final String WEKA_DB_DEFAULT_DIRECTORY = "wekaDB";
+    protected static final String KNN_FEATURES_FILENAME = "av-features.csv";
+
     
     protected WekaDBWrapper wekadb_src = null;
 
-    public GS2WekaAVRecommender() {
-    
+    public GS2WekaDBSearch() {
+
     if(this.wekadb_src == null) {
+        logger.info("Initializing WekaDBWrapper");
         this.wekadb_src = new WekaDBWrapper();
         }
@@ -66 +71 @@
     /** do the actual query */
     protected Element processAudioQuery (Element request) {
-    // MG needs to be synchronized (this inspiration for this class)
-    // Since it is not known how concurrent Weka can be, play it safe for
-    // now and also require synchronization
-
+
+    // As the MG version needs to be java-synchronized (this inspiration for this class)
+    // And since it is not known how concurrent (thread-safe) Weka can be ...
+    // => Play it safe for now and restrict access to 'wekadb_src' using synchronized also
         synchronized(this.wekadb_src) {
         // Create a new (empty) result message ('doc' is in ServiceRack.java)
@@ -99 +104 @@
         String index = (String) params.get (INDEX_PARAM);
         if (index == null) {
-        index = WEKA_MODEL_DEFAULT_DIRECTORY; 
-        }
-        
-        // The location of the Weka model index
+        index = WEKA_DB_DEFAULT_DIRECTORY; 
+        }
+        
+        // The location of the Weka db index
         String toplevel_index_dir = GSFile.collectionIndexDir (this.site_home, this.cluster_name);
-        String weka_model_index_dir = toplevel_index_dir + File.separatorChar + index;
+        String weka_db_index_dir = toplevel_index_dir + File.separatorChar + index;
         String assoc_index_dir = toplevel_index_dir + File.separatorChar + "assoc"; // ****
 
-        // set the Weka model query parameters to the values the user has specified
+        // set the Weka DB query parameters to the values the user has specified
         setStandardQueryParams (params); // ****
         
-        this.wekadb_src.runQuery (weka_model_index_dir, KNN_MODEL_FILENAME, assoc_index_dir, query);
+        this.wekadb_src.runQuery (weka_db_index_dir, KNN_FEATURES_FILENAME, assoc_index_dir, query);
         Vector docs = this.wekadb_src.getQueryResult ();
 
         if (docs.isEmpty()) {
         // something has gone wrong
-        GSXML.addError (result, "Couldn't query the Weka model", GSXML.ERROR_TYPE_SYSTEM);
+        GSXML.addError (result, "Couldn't query the Weka DB", GSXML.ERROR_TYPE_SYSTEM);
         return result;
         }

gs3-extensions/mars-src/trunk/src/java/org/greenstone/gsdl3/util/WekaDBWrapper.java

@@ -22 +22 @@
 import java.util.Vector;
 import java.util.Collections;
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+
+import weka.core.Attribute;
+import weka.core.Instance;
+import weka.core.Instances;
 
 import org.apache.log4j.*;
+
+import org.greenstone.gsdl3.util.WekaFindInstanceKNN;
 
 /** Java wrapper class for access to the Weka 
@@ -107 +115 @@
      * base_dir must end with a file separator (OS dependant)
      */
+
+
     public void runQuery(String wekaDB_index_dir, String knn_model_file, 
+             String assoc_index_dir, String query_string) {
+
+    // combine index_dir with audiodb fileanem
+
+    String full_knn_model_filename  = wekaDB_index_dir + File.separatorChar + knn_model_file;
+
+    //String full_chr12_filename = assoc_index_dir + File.separatorChar 
+    //    + query_string + File.separatorChar + "doc.chr12";
+
+    System.err.println("**** full knn model filename  = " + full_knn_model_filename);
+
+    // Example returned result from Weka KNN
+    // => first line is the input instance ('filename+segment',Arousal,Valence)
+    //    following (indented lines) nearest neighbour matches in same format
+    //
+    // ds_22716_5743-6,-0.549489,-0.118439
+    //  ds_22716_5743-6,-0.549489,-0.118439
+    //  ds_31008_6550-30,-0.549489,-0.118439
+    //  ds_72651_26831-6,-0.549489,-0.118439
+    //  ds_26196_9214-18,-0.549489,-0.118439
+
+
+    WekaFindInstanceKNN.init(full_knn_model_filename);
+
+    //Instances nearest_instances = WekaFindInstanceKNN.kNearestNeighbours("ds_22716_5743",6,-0.549489,-0.118439,3);
+    String doc_id = query_string;
+
+    int segment = 6;
+    //String segment_str = Integer.toString(segment);
+    String doc_id_segment =  doc_id +"-" + segment;
+
+    double arousal_val = -0.549489;
+    double valence_val =  -0.118439;
+    int k_nearest_num = 3;
+    
+    System.err.println("doc_id_segment = " + doc_id_segment);
+
+    
+    Instances nearest_instances = WekaFindInstanceKNN.kNearestNeighbours(doc_id_segment,arousal_val,valence_val,k_nearest_num);
+
+        
+    query_result_ = new Vector();
+
+    /*
+    WekaDBDocInfo wekaDB_doc_info1 = new WekaDBDocInfo("ds_20415_2337",0.9,4);
+    query_result_.add(wekaDB_doc_info1);
+
+    WekaDBDocInfo wekaDB_doc_info2 = new WekaDBDocInfo("ds_51017_15513",0.87,1);
+    query_result_.add(wekaDB_doc_info2);
+
+    WekaDBDocInfo wekaDB_doc_info3 = new WekaDBDocInfo("ds_20415_2337",0.82,6);
+    query_result_.add(wekaDB_doc_info3);
+    */
+
+    int clamped_k_nearest_num = Math.min(k_nearest_num,nearest_instances.size());
+    
+    for (int i=0; i<clamped_k_nearest_num; i++) {
+        Instance instance = nearest_instances.instance(i);
+        logger.info("\tProcessing returned instance: " + instance);
+        
+        //Attribute doc_id_segment_att = instance.attribute(0);
+        //String matching_doc_id_segment = instance.attribute(0).value(0);
+        String matching_doc_id_segment = instance.stringValue(0);
+        //double arousal_val    = instance.attribute(1);
+        //double valence_val    = instance.attribute(2);
+
+
+        Pattern p = Pattern.compile("^(\\w+)-(\\d+)$");
+        Matcher m = p.matcher(matching_doc_id_segment);
+        if (m.matches()) {
+
+        String matching_doc_id = m.group(1);
+        int matching_segment_offset = Integer.parseInt(m.group(2));
+        
+        double matching_rank = 0.9;
+
+        logger.info("\tAdding in: matching_doc_id = " + matching_doc_id);
+        WekaDBDocInfo wekaDB_doc_info = new WekaDBDocInfo(matching_doc_id,matching_rank,matching_segment_offset);
+        query_result_.add(wekaDB_doc_info);
+        }
+        else {
+        logger.error("Returned AV k-nearest neighbour match '"+doc_id_segment+"' could not be parsed as <doc-id>-<segment>" );
+        }
+    }
+    
+    /*
+    int num_matches_within_track = 6;
+
+
+    first_entry = addQueryResult(first_entry,root_doc_id,rankVector,offsetVector);
+    // and now reset vectors to empty to be ready for next chain of values
+    rankVector = new Vector<Double>();
+    offsetVector = new Vector<Integer>();
+
+
+    rankVector.add(rank);
+    offsetVector.add(target_frame);
+    
+    addQueryResult(first_entry,root_doc_id,rankVector,offsetVector);
+    */
+
+    }
+    
+    public void runQueryOLD(String wekaDB_index_dir, String knn_model_file, 
              String assoc_index_dir, String query_string) {
 
@@ -120 +234 @@
     // ****
     String [] cmd_array = new String[] {
-        "java -jar weka.jar",
+        "java", "-jar", "weka.jar",
         "-d", full_knn_model_filename,
         "-Q", "nsequence",