Index: other-projects/mars-music-recommender/trunk/amc-music-scrape/src/AMCArtistScrape.java
===================================================================
--- other-projects/mars-music-recommender/trunk/amc-music-scrape/src/AMCArtistScrape.java	(revision 30424)
+++ other-projects/mars-music-recommender/trunk/amc-music-scrape/src/AMCArtistScrape.java	(revision 30425)
@@ -3,4 +3,5 @@
 import java.net.*;
 import java.nio.*;
+import java.nio.charset.*;
 import java.nio.channels.*;
 import java.nio.file.*;
@@ -21,14 +22,54 @@
     }
 
-    public static void downloadURL(String url_str) {
+    public static File urlToLocalFile(String url_str,String ext) 
+    {
+
+	String local_file_str = url_str.substring(url_str.lastIndexOf(File.separator)+1, url_str.length() );
+
+	int ext_dot_pos = local_file_str.lastIndexOf('.');
+	String local_file_root_str = local_file_str;
+	if (ext_dot_pos>0) {
+	    local_file_root_str = local_file_str.substring(0,ext_dot_pos);
+	}
+	
+	if (ext != null) {
+	    local_file_str = local_file_root_str + ext;
+	}
+
+	// Add in 'repair' style subdirectory
+	String repair_subdir = local_file_root_str.substring(local_file_root_str.length()-2);
+
+	File local_dir = new File("audio",repair_subdir);
+	File local_file = null;
+	try {
+	    if (!local_dir.exists()) {
+		local_dir.mkdir();
+	    }
+
+	    local_file = new File(local_dir,local_file_str);
+	}
+	catch (Exception e) {
+	    e.printStackTrace();
+	}
+
+	return local_file;
+    }
+
+    public static File urlToLocalFile(String url_str) 
+    {
+	return urlToLocalFile(url_str,null);
+    }
+
+
+    public static void downloadURL(String url_str) 
+    {
 
 	try {
 	    URL url = new URL(url_str);
 
-	    String local_file_str = url_str.substring(url_str.lastIndexOf('/')+1, url_str.length() );
-	    File local_file = new File("audio",local_file_str);
+	    File local_file = urlToLocalFile(url_str);
 
 	    if (!local_file.exists()) {
-		System.out.printf("Downloading audio to '%s'...\n",local_file_str);
+		System.out.printf("Downloading audio to '%s'...\n",local_file.getPath());
 		FileOutputStream fos = new FileOutputStream(local_file);
 
@@ -39,13 +80,27 @@
 	    }
 	    else {
-		System.out.printf("Skipping '%s' as is already exists\n",local_file_str);
-	    }
-	    
-	}
-	catch (Exception e) {
-	    e.printStackTrace();
-	}
-    }
-    
+		System.out.printf("Skipping '%s' as is already exists\n",local_file.getPath());
+	    }
+	    
+	}
+	catch (Exception e) {
+	    e.printStackTrace();
+	}
+    }
+
+    public static void saveJSONMetadata(MusicRecord mr, String url_str) 
+    {
+	File json_file = urlToLocalFile(url_str,".json");
+	String json_text = mr.toJSONString();
+
+	try {
+	    Files.write(Paths.get(json_file.getAbsolutePath()),json_text.getBytes(StandardCharsets.UTF_8));
+	}
+	catch (IOException e) {
+	    e.printStackTrace();
+	}
+
+    }
+
     public static ArrayList<MusicRecord> extractArtistRecordsFromPage(Document doc)
     {
@@ -72,5 +127,5 @@
 		    
 		    MusicRecord mr = new MusicRecord(track_url,title,composer,year,audio_url);
-		    System.out.println(mr.toJSONString());
+		    //System.out.println(mr.toJSONString());
 		    
 		    records.add(mr);
@@ -84,7 +139,8 @@
     }
 
-    private static void followArtistPages(String base_domain,String base_url, String artist_suffix)
-    {
-	String artist_url_str = base_domain+base_url+artist_suffix;
+    //private static void followArtistPages(String base_domain,String base_url, String artist_suffix)
+    private static void followArtistPages(String base_domain,String starting_href)
+    {
+	String artist_works_url_str = base_domain+starting_href;
 
 	try {
@@ -93,12 +149,18 @@
 
 	    while (has_next_page) {
-		System.out.println("Procesing Artist suffix '" + artist_suffix + "': Page " + page_number);
-
-		Document doc = Jsoup.connect(artist_url_str).get();
+		System.out.println("Procesing Artist URL " + artist_works_url_str + ": Page " + page_number);
+
+		Document doc = Jsoup.connect(artist_works_url_str).get();
 
 		ArrayList<MusicRecord> music_records = extractArtistRecordsFromPage(doc);
 		
 		for (MusicRecord mr: music_records) {
-		    downloadURL(mr.getAudioURL());
+		    String audio_url = mr.getAudioURL();
+		    downloadURL(audio_url);
+
+		    saveJSONMetadata(mr,audio_url);
+
+		    //System.out.println(mr.toJSONString());
+
 		}
 
@@ -110,7 +172,13 @@
 		Elements pagination_block = doc.select("ul.pagination > li > a");
 		Element last_li_a = pagination_block.last();
-		String last_li_a_text = last_li_a.text();
-		if (last_li_a_text.equals("Next >")) {
-		    artist_url_str = base_domain+last_li_a.attr("href");
+
+		if (last_li_a != null) {
+		    String last_li_a_text = last_li_a.text();
+		    if (last_li_a_text.equals("Next >")) {
+			artist_works_url_str = base_domain+last_li_a.attr("href");
+		    }
+		    else {
+			has_next_page = false;
+		    }
 		}
 		else {
@@ -119,7 +187,4 @@
 
 		page_number++;
-
-		//System.out.println("*** last li a text = " + last_li_a_text);
-		//System.out.println("*** artist_url_str = " + artist_url_str);
 	    }
 	}
@@ -130,6 +195,6 @@
 
     
-
-    public static void main(String[] args)
+    /*
+    public static void pageScrapeArtist(String artist_suffix)
     {
 
@@ -141,5 +206,49 @@
 	base_url += "&wfc[]=";
 
-	followArtistPages(base_domain,base_url,"Roger+Dean");
+	followArtistPages(base_domain,base_url+"Roger+Dean");
+    }
+    */
+
+
+    public static void main(String[] args)
+    {
+
+	// Represented artists at AMC
+	//   http://www.australianmusiccentre.com.au/artists
+
+	String base_domain = "http://www.australianmusiccentre.com.au";
+	String base_url = "/artists";
+	
+	String represented_artists_url_str = base_domain+base_url;
+
+	try {
+	    Document doc = Jsoup.connect(represented_artists_url_str).get();
+
+	    Elements artist_paras = doc.select("p.artist_home_name");
+	    
+	    for (Element ap : artist_paras) {
+		
+		Element artist_a = ap.select("a").first();
+		if (artist_a != null) {
+		    String artist_about_url_str = base_domain + artist_a.attr("href");
+		    
+		    
+		    // Now process about this artist page to get to the sequence of pages
+		    Document about_doc = Jsoup.connect(artist_about_url_str).get();
+
+		    Element browse_by_artist = about_doc.select("#content > div.inner > ul > li > a").first();
+		    if (browse_by_artist != null) {
+
+			String browse_works_href= browse_by_artist.attr("href");
+			
+			followArtistPages(base_domain,browse_works_href);
+		    }
+		}
+	    }
+	}
+	catch (Exception e) {
+	    e.printStackTrace();
+	}
+	
     }
 }
