Index: /main/trunk/model-sites-dev/mars/collect/amc-essentia/prepare/COMPILE.sh
===================================================================
--- /main/trunk/model-sites-dev/mars/collect/amc-essentia/prepare/COMPILE.sh	(revision 34365)
+++ /main/trunk/model-sites-dev/mars/collect/amc-essentia/prepare/COMPILE.sh	(revision 34365)
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+
+cygpath -h >/dev/null 2>&1 
+if [ $? = "0" ] ; then
+    is_cygwin=1
+else
+    is_cygwin=0
+fi
+
+if [ ! -d bin ] ; then
+  echo "Creating directory 'bin' for compiled Java code"
+  mkdir bin
+fi
+
+if [ $is_cygwin = "1" ] ; then
+    
+    echo javac -cp jars\\jsoup-1.8.3.jar\;jars\\json-simple-1.1.1.jar\;src -d bin src\\AMCArtistScrape.java
+    javac -cp jars\\jsoup-1.8.3.jar\;jars\\json-simple-1.1.1.jar\;src -d bin src\\AMCArtistScrape.java
+else
+    
+    echo javac -cp jars/jsoup-1.8.3.jar:jars/json-simple-1.1.1.jar:src -d bin src/AMCArtistScrape.java
+    javac -cp jars/jsoup-1.8.3.jar:jars/json-simple-1.1.1.jar:src -d bin src/AMCArtistScrape.java
+
+fi
+
+
Index: /main/trunk/model-sites-dev/mars/collect/amc-essentia/prepare/RUN.sh
===================================================================
--- /main/trunk/model-sites-dev/mars/collect/amc-essentia/prepare/RUN.sh	(revision 34365)
+++ /main/trunk/model-sites-dev/mars/collect/amc-essentia/prepare/RUN.sh	(revision 34365)
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+cygpath -h >/dev/null 2>&1 
+if [ $? = "0" ] ; then
+    is_cygwin=1
+else
+    is_cygwin=0
+fi
+
+if [ ! -d audio ] ; then
+  echo "Creating directory 'audio' for downloaded audio files and JSON metadata"
+  mkdir audio
+fi
+
+
+if [ $is_cygwin = "1" ] ; then
+    java -cp jars\\jsoup-1.8.3.jar\;jars\\json-simple-1.1.1.jar\;bin AMCArtistScrape
+else
+    java -cp jars/jsoup-1.8.3.jar:jars/json-simple-1.1.1.jar:bin AMCArtistScrape    
+fi
+
+    
Index: /main/trunk/model-sites-dev/mars/collect/amc-essentia/prepare/src/AMCArtistScrape.java
===================================================================
--- /main/trunk/model-sites-dev/mars/collect/amc-essentia/prepare/src/AMCArtistScrape.java	(revision 34365)
+++ /main/trunk/model-sites-dev/mars/collect/amc-essentia/prepare/src/AMCArtistScrape.java	(revision 34365)
@@ -0,0 +1,254 @@
+
+import java.io.*;
+import java.net.*;
+import java.nio.*;
+import java.nio.charset.*;
+import java.nio.channels.*;
+import java.nio.file.*;
+import java.util.*;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+
+class AMCArtistScrape
+{
+
+    private static void printf(String msg, Object... args)
+    {
+	System.out.println(String.format(msg, args));
+    }
+
+    public static File urlToLocalFile(String url_str,String ext) 
+    {
+
+	String local_file_str = url_str.substring(url_str.lastIndexOf(File.separator)+1, url_str.length() );
+
+	int ext_dot_pos = local_file_str.lastIndexOf('.');
+	String local_file_root_str = local_file_str;
+	if (ext_dot_pos>0) {
+	    local_file_root_str = local_file_str.substring(0,ext_dot_pos);
+	}
+	
+	if (ext != null) {
+	    local_file_str = local_file_root_str + ext;
+	}
+
+	// Add in 'repair' style subdirectory
+	String repair_subdir = local_file_root_str.substring(local_file_root_str.length()-2);
+
+	File local_dir = new File("audio",repair_subdir);
+	File local_file = null;
+	try {
+	    if (!local_dir.exists()) {
+		local_dir.mkdir();
+	    }
+
+	    local_file = new File(local_dir,local_file_str);
+	}
+	catch (Exception e) {
+	    e.printStackTrace();
+	}
+
+	return local_file;
+    }
+
+    public static File urlToLocalFile(String url_str) 
+    {
+	return urlToLocalFile(url_str,null);
+    }
+
+
+    public static void downloadURL(String url_str) 
+    {
+
+	try {
+	    URL url = new URL(url_str);
+
+	    File local_file = urlToLocalFile(url_str);
+
+	    if (!local_file.exists()) {
+		System.out.printf("Downloading audio to '%s'...\n",local_file.getPath());
+		FileOutputStream fos = new FileOutputStream(local_file);
+
+		ReadableByteChannel rbc = Channels.newChannel(url.openStream());
+		fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+
+		System.out.println("... done");
+	    }
+	    else {
+		System.out.printf("Skipping '%s' as is already exists\n",local_file.getPath());
+	    }
+	    
+	}
+	catch (Exception e) {
+	    e.printStackTrace();
+	}
+    }
+
+    public static void saveJSONMetadata(MusicRecord mr, String url_str) 
+    {
+	File json_file = urlToLocalFile(url_str,".json");
+	String json_text = mr.toJSONString();
+
+	try {
+	    Files.write(Paths.get(json_file.getAbsolutePath()),json_text.getBytes(StandardCharsets.UTF_8));
+	}
+	catch (IOException e) {
+	    e.printStackTrace();
+	}
+
+    }
+
+    public static ArrayList<MusicRecord> extractArtistRecordsFromPage(Document doc)
+    {
+
+	ArrayList<MusicRecord> records = new ArrayList<MusicRecord>();
+	
+	Elements table_rows = doc.select("table.search_results tr");
+	
+	for (Element tr : table_rows) {
+	    
+	    Elements td_cells = tr.select("td");
+	    
+	    if (td_cells.size() == 6) {
+		Element title_anchor = td_cells.get(0).select("a").first();
+		String track_url       = title_anchor.attr("href");
+		String title           = title_anchor.ownText();
+		String instrumentation = td_cells.get(1).ownText();
+		String composer        = td_cells.get(2).ownText();
+		String year            = td_cells.get(3).ownText();
+		
+		Element audio_elem = td_cells.get(4).select("audio").first();
+		if (audio_elem != null) {
+		    String audio_url = audio_elem.attr("src");
+		    
+		    MusicRecord mr = new MusicRecord(track_url,title,composer,year,audio_url);
+		    //System.out.println(mr.toJSONString());
+		    
+		    records.add(mr);
+		}
+		
+	    }					   
+	    
+	}
+	
+	return records;
+    }
+
+    //private static void followArtistPages(String base_domain,String base_url, String artist_suffix)
+    private static void followArtistPages(String base_domain,String starting_href)
+    {
+	String artist_works_url_str = base_domain+starting_href;
+
+	try {
+	    boolean has_next_page = true;
+	    int page_number = 1;
+
+	    while (has_next_page) {
+		System.out.println("Procesing Artist URL " + artist_works_url_str + ": Page " + page_number);
+
+		Document doc = Jsoup.connect(artist_works_url_str).get();
+
+		ArrayList<MusicRecord> music_records = extractArtistRecordsFromPage(doc);
+		
+		for (MusicRecord mr: music_records) {
+		    String audio_url = mr.getAudioURL();
+		    downloadURL(audio_url);
+
+		    saveJSONMetadata(mr,audio_url);
+
+		    //System.out.println(mr.toJSONString());
+
+		}
+
+		// <ul class="pagination">
+		//  ...
+		//  <li href="URL-TO-FOLLOW">Next</li>
+		// </ul>
+
+		Elements pagination_block = doc.select("ul.pagination > li > a");
+		Element last_li_a = pagination_block.last();
+
+		if (last_li_a != null) {
+		    String last_li_a_text = last_li_a.text();
+		    if (last_li_a_text.equals("Next >")) {
+			artist_works_url_str = base_domain+last_li_a.attr("href");
+		    }
+		    else {
+			has_next_page = false;
+		    }
+		}
+		else {
+		    has_next_page = false;
+		}
+
+		page_number++;
+	    }
+	}
+	catch (Exception e) {
+	    e.printStackTrace();
+	}
+    }
+
+    
+    /*
+    public static void pageScrapeArtist(String artist_suffix)
+    {
+
+	// Example artist page at AMC
+	//   http://www.australianmusiccentre.com.au/search?type=work&sort=alphaTitleSort&wfc[]=Roger+Dean
+
+	String base_domain = "http://www.australianmusiccentre.com.au";
+	String base_url = "/search?type=work&sort=alphaTitleSort";
+	base_url += "&wfc[]=";
+
+	followArtistPages(base_domain,base_url+"Roger+Dean");
+    }
+    */
+
+
+    public static void main(String[] args)
+    {
+
+	// Represented artists at AMC
+	//   http://www.australianmusiccentre.com.au/artists
+
+	String base_domain = "http://www.australianmusiccentre.com.au";
+	String base_url = "/artists";
+	
+	String represented_artists_url_str = base_domain+base_url;
+
+	try {
+	    Document doc = Jsoup.connect(represented_artists_url_str).get();
+
+	    Elements artist_paras = doc.select("p.artist_home_name");
+	    
+	    for (Element ap : artist_paras) {
+		
+		Element artist_a = ap.select("a").first();
+		if (artist_a != null) {
+		    String artist_about_url_str = base_domain + artist_a.attr("href");
+		    
+		    
+		    // Now process about this artist page to get to the sequence of pages
+		    Document about_doc = Jsoup.connect(artist_about_url_str).get();
+
+		    Element browse_by_artist = about_doc.select("#content > div.inner > ul > li > a").first();
+		    if (browse_by_artist != null) {
+
+			String browse_works_href= browse_by_artist.attr("href");
+			
+			followArtistPages(base_domain,browse_works_href);
+		    }
+		}
+	    }
+	}
+	catch (Exception e) {
+	    e.printStackTrace();
+	}
+	
+    }
+}
Index: /main/trunk/model-sites-dev/mars/collect/amc-essentia/prepare/src/MusicRecord.java
===================================================================
--- /main/trunk/model-sites-dev/mars/collect/amc-essentia/prepare/src/MusicRecord.java	(revision 34365)
+++ /main/trunk/model-sites-dev/mars/collect/amc-essentia/prepare/src/MusicRecord.java	(revision 34365)
@@ -0,0 +1,96 @@
+
+import java.io.*;
+import java.net.*;
+import java.nio.*;
+import java.nio.channels.*;
+import java.nio.file.*;
+import java.util.*;
+
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+
+class MusicRecord {
+    private String track_url_;
+    private String title_;
+    private String artist_;
+    private String year_;
+    private String audio_url_;
+    
+    public MusicRecord(String track_url,String title, String artist, String year, String audio_url) {
+	track_url_ = track_url;
+	title_  = title;
+	artist_ = artist;
+	year_   = (year.matches("^\\s*$")) ? null : year;
+	audio_url_  = audio_url;
+    }
+
+    public String getAudioURL() {
+	return audio_url_;
+    }
+    
+    public String toString()
+    {
+	
+	return String.format("[title=%s, artist=%s, year=%s; track_url=%s, audio_url=%s]",
+			     title_,artist_,year_, track_url_, audio_url_);
+    }
+
+    public JSONObject toJSON() 
+    {
+	JSONObject obj = new JSONObject();
+
+	boolean object_is_all_null = true;
+
+	if (title_ != null) {
+	    obj.put("title",title_);
+	    object_is_all_null = false;
+	}
+
+	if (artist_ != null) {
+	    obj.put("artist",artist_);
+	    object_is_all_null = false;
+	}
+
+	if (year_ != null) {
+	    obj.put("year",year_);
+	    object_is_all_null = false;
+	}
+
+	if (track_url_ != null) {
+	    obj.put("track_url",track_url_);
+	    object_is_all_null = false;
+	}
+
+	if (audio_url_ != null) {
+	    obj.put("audio_url",audio_url_);
+	    object_is_all_null = false;
+	}
+
+	if (object_is_all_null) {
+	    System.err.println("Warning: MusicRecord '" + this + "' only has null values in it");
+	}
+
+	return obj;
+    }
+
+    public String toJSONString() 
+    {
+	String jsonText = null;
+	JSONObject obj = toJSON();
+
+	try {
+	    StringWriter out = new StringWriter();
+	    obj.writeJSONString(out);
+      
+	    jsonText = out.toString();
+	}
+	catch (IOException ioe) {
+	    ioe.printStackTrace();
+	}
+
+	return jsonText;
+    }
+
+
+}
+
