Changeset 30425

Timestamp:

2016-03-21T21:15:13+13:00 (8 years ago)

Author:

davidb

Message:

Save metadata as JSON file. Create sub-directories to spreadout the generated files

File:

• other-projects/mars-music-recommender/trunk/amc-music-scrape/src/AMCArtistScrape.java (modified) (10 diffs)

other-projects/mars-music-recommender/trunk/amc-music-scrape/src/AMCArtistScrape.java

@@ -3 +3 @@
 import java.net.*;
 import java.nio.*;
+import java.nio.charset.*;
 import java.nio.channels.*;
 import java.nio.file.*;
@@ -21 +22 @@
     }
 
-    public static void downloadURL(String url_str) {
+    public static File urlToLocalFile(String url_str,String ext) 
+    {
+
+    String local_file_str = url_str.substring(url_str.lastIndexOf(File.separator)+1, url_str.length() );
+
+    int ext_dot_pos = local_file_str.lastIndexOf('.');
+    String local_file_root_str = local_file_str;
+    if (ext_dot_pos>0) {
+        local_file_root_str = local_file_str.substring(0,ext_dot_pos);
+    }
+    
+    if (ext != null) {
+        local_file_str = local_file_root_str + ext;
+    }
+
+    // Add in 'repair' style subdirectory
+    String repair_subdir = local_file_root_str.substring(local_file_root_str.length()-2);
+
+    File local_dir = new File("audio",repair_subdir);
+    File local_file = null;
+    try {
+        if (!local_dir.exists()) {
+        local_dir.mkdir();
+        }
+
+        local_file = new File(local_dir,local_file_str);
+    }
+    catch (Exception e) {
+        e.printStackTrace();
+    }
+
+    return local_file;
+    }
+
+    public static File urlToLocalFile(String url_str) 
+    {
+    return urlToLocalFile(url_str,null);
+    }
+
+
+    public static void downloadURL(String url_str) 
+    {
 
     try {
         URL url = new URL(url_str);
 
-        String local_file_str = url_str.substring(url_str.lastIndexOf('/')+1, url_str.length() );
-        File local_file = new File("audio",local_file_str);
+        File local_file = urlToLocalFile(url_str);
 
         if (!local_file.exists()) {
-        System.out.printf("Downloading audio to '%s'...\n",local_file_str);
+        System.out.printf("Downloading audio to '%s'...\n",local_file.getPath());
         FileOutputStream fos = new FileOutputStream(local_file);
 
@@ -39 +80 @@
         }
         else {
-        System.out.printf("Skipping '%s' as is already exists\n",local_file_str);
-        }
-        
-    }
-    catch (Exception e) {
-        e.printStackTrace();
-    }
-    }
-    
+        System.out.printf("Skipping '%s' as is already exists\n",local_file.getPath());
+        }
+        
+    }
+    catch (Exception e) {
+        e.printStackTrace();
+    }
+    }
+
+    public static void saveJSONMetadata(MusicRecord mr, String url_str) 
+    {
+    File json_file = urlToLocalFile(url_str,".json");
+    String json_text = mr.toJSONString();
+
+    try {
+        Files.write(Paths.get(json_file.getAbsolutePath()),json_text.getBytes(StandardCharsets.UTF_8));
+    }
+    catch (IOException e) {
+        e.printStackTrace();
+    }
+
+    }
+
     public static ArrayList<MusicRecord> extractArtistRecordsFromPage(Document doc)
     {
@@ -72 +127 @@
             
             MusicRecord mr = new MusicRecord(track_url,title,composer,year,audio_url);
-            System.out.println(mr.toJSONString());
+            //System.out.println(mr.toJSONString());
             
             records.add(mr);
@@ -84 +139 @@
     }
 
-    private static void followArtistPages(String base_domain,String base_url, String artist_suffix)
-    {
-    String artist_url_str = base_domain+base_url+artist_suffix;
+    //private static void followArtistPages(String base_domain,String base_url, String artist_suffix)
+    private static void followArtistPages(String base_domain,String starting_href)
+    {
+    String artist_works_url_str = base_domain+starting_href;
 
     try {
@@ -93 +149 @@
 
         while (has_next_page) {
-        System.out.println("Procesing Artist suffix '" + artist_suffix + "': Page " + page_number);
-
-        Document doc = Jsoup.connect(artist_url_str).get();
+        System.out.println("Procesing Artist URL " + artist_works_url_str + ": Page " + page_number);
+
+        Document doc = Jsoup.connect(artist_works_url_str).get();
 
         ArrayList<MusicRecord> music_records = extractArtistRecordsFromPage(doc);
         
         for (MusicRecord mr: music_records) {
-            downloadURL(mr.getAudioURL());
+            String audio_url = mr.getAudioURL();
+            downloadURL(audio_url);
+
+            saveJSONMetadata(mr,audio_url);
+
+            //System.out.println(mr.toJSONString());
+
         }
 
@@ -110 +172 @@
         Elements pagination_block = doc.select("ul.pagination > li > a");
         Element last_li_a = pagination_block.last();
-        String last_li_a_text = last_li_a.text();
-        if (last_li_a_text.equals("Next >")) {
-            artist_url_str = base_domain+last_li_a.attr("href");
+
+        if (last_li_a != null) {
+            String last_li_a_text = last_li_a.text();
+            if (last_li_a_text.equals("Next >")) {
+            artist_works_url_str = base_domain+last_li_a.attr("href");
+            }
+            else {
+            has_next_page = false;
+            }
         }
         else {
@@ -119 +187 @@
 
         page_number++;
-
-        //System.out.println("*** last li a text = " + last_li_a_text);
-        //System.out.println("*** artist_url_str = " + artist_url_str);
         }
     }
@@ -130 +195 @@
 
     
-
-    public static void main(String[] args)
+    /*
+    public static void pageScrapeArtist(String artist_suffix)
     {
 
@@ -141 +206 @@
     base_url += "&wfc[]=";
 
-    followArtistPages(base_domain,base_url,"Roger+Dean");
+    followArtistPages(base_domain,base_url+"Roger+Dean");
+    }
+    */
+
+
+    public static void main(String[] args)
+    {
+
+    // Represented artists at AMC
+    //   http://www.australianmusiccentre.com.au/artists
+
+    String base_domain = "http://www.australianmusiccentre.com.au";
+    String base_url = "/artists";
+    
+    String represented_artists_url_str = base_domain+base_url;
+
+    try {
+        Document doc = Jsoup.connect(represented_artists_url_str).get();
+
+        Elements artist_paras = doc.select("p.artist_home_name");
+        
+        for (Element ap : artist_paras) {
+        
+        Element artist_a = ap.select("a").first();
+        if (artist_a != null) {
+            String artist_about_url_str = base_domain + artist_a.attr("href");
+            
+            
+            // Now process about this artist page to get to the sequence of pages
+            Document about_doc = Jsoup.connect(artist_about_url_str).get();
+
+            Element browse_by_artist = about_doc.select("#content > div.inner > ul > li > a").first();
+            if (browse_by_artist != null) {
+
+            String browse_works_href= browse_by_artist.attr("href");
+            
+            followArtistPages(base_domain,browse_works_href);
+            }
+        }
+        }
+    }
+    catch (Exception e) {
+        e.printStackTrace();
+    }
+    
     }
 }