source: other-projects/mars-music-recommender/trunk/amc-music-scrape/src/AMCArtistScrape.java@ 30417

import java.io.*;
import java.net.*;
import java.nio.*;
import java.nio.channels.*;
import java.nio.file.*;
import java.util.*;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


class AMCArtistScrape
{

    public static void downloadURL(String url_str) {

    try {
        URL url = new URL(url_str);

        String local_file_str = url_str.substring(url_str.lastIndexOf('/')+1, url_str.length() );
        File local_file = new File(local_file_str);

        if (!local_file.exists()) {
        System.out.printf("Downloading audio to '%s'...\n",local_file_str);
        FileOutputStream fos = new FileOutputStream(local_file);

        ReadableByteChannel rbc = Channels.newChannel(url.openStream());
        fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);

        System.out.println("... done");
        }
        else {
        System.out.printf("Skipping '%s' as is already exists\n",local_file_str);
        }
        
    }
    catch (Exception e) {
        e.printStackTrace();
    }
    }
    
    public static ArrayList<MusicRecord> extractResults(String urlstr)
    {

    ArrayList<MusicRecord> records = new ArrayList<MusicRecord>();
    
    try {
        Document doc = Jsoup.connect(urlstr).get();
        
        Elements table_rows = doc.select("table.search_results tr");

        
        
        for (Element tr : table_rows) {

        Elements td_cells = tr.select("td");

        if (td_cells.size() == 6) {
            Element title_anchor = td_cells.get(0).select("a").first();
            String track_url       = title_anchor.attr("href");
            String title           = title_anchor.ownText();
            String instrumentation = td_cells.get(1).ownText();
            String composer        = td_cells.get(2).ownText();
            String year            = td_cells.get(3).ownText();

            Element audio_elem = td_cells.get(4).select("audio").first();
            if (audio_elem != null) {
            String audio_url = audio_elem.attr("src");

            MusicRecord mr = new MusicRecord(track_url,title,composer,year,audio_url);
            System.out.println(mr);

            records.add(mr);
            }
            
        }                      
        
        }
    }
    catch (Exception e) {
        e.printStackTrace();
    }

    return records;
    }

    private static void printf(String msg, Object... args)
    {
    System.out.println(String.format(msg, args));
    }
    

    public static void main(String[] args)
    {

    // http://www.australianmusiccentre.com.au/search?type=work&sort=alphaTitleSort&wfc[]=Roger+Dean
    String baseUrl = "http://www.australianmusiccentre.com.au/search?type=work&sort=alphaTitleSort";
    baseUrl += "&wfc[]=";

    ArrayList<MusicRecord> music_records = extractResults(baseUrl+"Roger+Dean");

    for (MusicRecord mr: music_records) {
        downloadURL(mr.getAudioURL());
    }
    }
}