Changeset 30418 for other-projects/mars-music-recommender

Timestamp:

2016-03-21T15:02:26+13:00 (8 years ago)

Author:

davidb

Message:

Code updated to work through a sequence of pages for one artist

Location:

other-projects/mars-music-recommender/trunk/amc-music-scrape/src

Files:

• AMCArtistScrape.java (modified) (5 diffs)
• MusicRecord.java (modified) (2 diffs)

other-projects/mars-music-recommender/trunk/amc-music-scrape/src/AMCArtistScrape.java

@@ -16 +16 @@
 {
 
+    private static void printf(String msg, Object... args)
+    {
+    System.out.println(String.format(msg, args));
+    }
+
     public static void downloadURL(String url_str) {
 
@@ -22 +27 @@
 
         String local_file_str = url_str.substring(url_str.lastIndexOf('/')+1, url_str.length() );
-        File local_file = new File(local_file_str);
+        File local_file = new File("audio",local_file_str);
 
         if (!local_file.exists()) {
@@ -43 +48 @@
     }
     
-    public static ArrayList<MusicRecord> extractResults(String urlstr)
+    public static ArrayList<MusicRecord> extractArtistRecordsFromPage(Document doc)
     {
 
     ArrayList<MusicRecord> records = new ArrayList<MusicRecord>();
     
+    Elements table_rows = doc.select("table.search_results tr");
+    
+    for (Element tr : table_rows) {
+        
+        Elements td_cells = tr.select("td");
+        
+        if (td_cells.size() == 6) {
+        Element title_anchor = td_cells.get(0).select("a").first();
+        String track_url       = title_anchor.attr("href");
+        String title           = title_anchor.ownText();
+        String instrumentation = td_cells.get(1).ownText();
+        String composer        = td_cells.get(2).ownText();
+        String year            = td_cells.get(3).ownText();
+        
+        Element audio_elem = td_cells.get(4).select("audio").first();
+        if (audio_elem != null) {
+            String audio_url = audio_elem.attr("src");
+            
+            MusicRecord mr = new MusicRecord(track_url,title,composer,year,audio_url);
+            System.out.println(mr.toJSONString());
+            
+            records.add(mr);
+        }
+        
+        }                      
+        
+    }
+    
+    return records;
+    }
+
+    private static void followArtistPages(String base_domain,String base_url, String artist_suffix)
+    {
+    String artist_url_str = base_domain+base_url+artist_suffix;
+
     try {
-        Document doc = Jsoup.connect(urlstr).get();
-        
-        Elements table_rows = doc.select("table.search_results tr");
+        boolean has_next_page = true;
+        int page_number = 1;
 
-        
-        
-        for (Element tr : table_rows) {
+        while (has_next_page) {
+        System.out.println("Procesing Artist suffix '" + artist_suffix + "': Page " + page_number);
 
-        Elements td_cells = tr.select("td");
+        Document doc = Jsoup.connect(artist_url_str).get();
 
-        if (td_cells.size() == 6) {
-            Element title_anchor = td_cells.get(0).select("a").first();
-            String track_url       = title_anchor.attr("href");
-            String title           = title_anchor.ownText();
-            String instrumentation = td_cells.get(1).ownText();
-            String composer        = td_cells.get(2).ownText();
-            String year            = td_cells.get(3).ownText();
+        ArrayList<MusicRecord> music_records = extractArtistRecordsFromPage(doc);
+        
+        for (MusicRecord mr: music_records) {
+            downloadURL(mr.getAudioURL());
+        }
 
-            Element audio_elem = td_cells.get(4).select("audio").first();
-            if (audio_elem != null) {
-            String audio_url = audio_elem.attr("src");
+        // <ul class="pagination">
+        //  ...
+        //  <li href="URL-TO-FOLLOW">Next</li>
+        // </ul>
 
-            MusicRecord mr = new MusicRecord(track_url,title,composer,year,audio_url);
-            System.out.println(mr);
+        Elements pagination_block = doc.select("ul.pagination > li > a");
+        Element last_li_a = pagination_block.last();
+        String last_li_a_text = last_li_a.text();
+        if (last_li_a_text.equals("Next >")) {
+            artist_url_str = base_domain+last_li_a.attr("href");
+        }
+        else {
+            has_next_page = false;
+        }
 
-            records.add(mr);
-            }
-            
-        }                      
-        
+        page_number++;
+
+        //System.out.println("*** last li a text = " + last_li_a_text);
+        //System.out.println("*** artist_url_str = " + artist_url_str);
         }
     }
@@ -84 +127 @@
         e.printStackTrace();
     }
-
-    return records;
     }
 
-    private static void printf(String msg, Object... args)
-    {
-    System.out.println(String.format(msg, args));
-    }
     
 
@@ -97 +134 @@
     {
 
-    // http://www.australianmusiccentre.com.au/search?type=work&sort=alphaTitleSort&wfc[]=Roger+Dean
-    String baseUrl = "http://www.australianmusiccentre.com.au/search?type=work&sort=alphaTitleSort";
-    baseUrl += "&wfc[]=";
+    // Example artist page at AMC
+    //   http://www.australianmusiccentre.com.au/search?type=work&sort=alphaTitleSort&wfc[]=Roger+Dean
 
-    ArrayList<MusicRecord> music_records = extractResults(baseUrl+"Roger+Dean");
+    String base_domain = "http://www.australianmusiccentre.com.au";
+    String base_url = "/search?type=work&sort=alphaTitleSort";
+    base_url += "&wfc[]=";
 
-    for (MusicRecord mr: music_records) {
-        downloadURL(mr.getAudioURL());
-    }
+    followArtistPages(base_domain,base_url,"Roger+Dean");
     }
 }

other-projects/mars-music-recommender/trunk/amc-music-scrape/src/MusicRecord.java

@@ -7 +7 @@
 import java.util.*;
 
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
 
 class MusicRecord {
@@ -33 +35 @@
                  title_,artist_,year_, track_url_, audio_url_);
     }
-    
+
+    public JSONObject toJSON() 
+    {
+    JSONObject obj = new JSONObject();
+
+    boolean object_is_all_null = true;
+
+    if (title_ != null) {
+        obj.put("title",title_);
+        object_is_all_null = false;
+    }
+
+    if (artist_ != null) {
+        obj.put("artist",artist_);
+        object_is_all_null = false;
+    }
+
+    if (year_ != null) {
+        obj.put("year",year_);
+        object_is_all_null = false;
+    }
+
+    if (track_url_ != null) {
+        obj.put("track_url",track_url_);
+        object_is_all_null = false;
+    }
+
+    if (audio_url_ != null) {
+        obj.put("audio_url",audio_url_);
+        object_is_all_null = false;
+    }
+
+    if (object_is_all_null) {
+        System.err.println("Warning: MusicRecord '" + this + "' only has null values in it");
+    }
+
+    return obj;
+    }
+
+    public String toJSONString() 
+    {
+    String jsonText = null;
+    JSONObject obj = toJSON();
+
+    try {
+        StringWriter out = new StringWriter();
+        obj.writeJSONString(out);
+      
+        jsonText = out.toString();
+    }
+    catch (IOException ioe) {
+        ioe.printStackTrace();
+    }
+
+    return jsonText;
+    }
+
+
 }