source: other-projects/mars-music-recommender/trunk/amc-music-scrape/src/AMCArtistScrape.java@ 30425

import java.io.*;
import java.net.*;
import java.nio.*;
import java.nio.charset.*;
import java.nio.channels.*;
import java.nio.file.*;
import java.util.*;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


class AMCArtistScrape
{

    private static void printf(String msg, Object... args)
    {
    System.out.println(String.format(msg, args));
    }

    public static File urlToLocalFile(String url_str,String ext) 
    {

    String local_file_str = url_str.substring(url_str.lastIndexOf(File.separator)+1, url_str.length() );

    int ext_dot_pos = local_file_str.lastIndexOf('.');
    String local_file_root_str = local_file_str;
    if (ext_dot_pos>0) {
        local_file_root_str = local_file_str.substring(0,ext_dot_pos);
    }
    
    if (ext != null) {
        local_file_str = local_file_root_str + ext;
    }

    // Add in 'repair' style subdirectory
    String repair_subdir = local_file_root_str.substring(local_file_root_str.length()-2);

    File local_dir = new File("audio",repair_subdir);
    File local_file = null;
    try {
        if (!local_dir.exists()) {
        local_dir.mkdir();
        }

        local_file = new File(local_dir,local_file_str);
    }
    catch (Exception e) {
        e.printStackTrace();
    }

    return local_file;
    }

    public static File urlToLocalFile(String url_str) 
    {
    return urlToLocalFile(url_str,null);
    }


    public static void downloadURL(String url_str) 
    {

    try {
        URL url = new URL(url_str);

        File local_file = urlToLocalFile(url_str);

        if (!local_file.exists()) {
        System.out.printf("Downloading audio to '%s'...\n",local_file.getPath());
        FileOutputStream fos = new FileOutputStream(local_file);

        ReadableByteChannel rbc = Channels.newChannel(url.openStream());
        fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);

        System.out.println("... done");
        }
        else {
        System.out.printf("Skipping '%s' as is already exists\n",local_file.getPath());
        }
        
    }
    catch (Exception e) {
        e.printStackTrace();
    }
    }

    public static void saveJSONMetadata(MusicRecord mr, String url_str) 
    {
    File json_file = urlToLocalFile(url_str,".json");
    String json_text = mr.toJSONString();

    try {
        Files.write(Paths.get(json_file.getAbsolutePath()),json_text.getBytes(StandardCharsets.UTF_8));
    }
    catch (IOException e) {
        e.printStackTrace();
    }

    }

    public static ArrayList<MusicRecord> extractArtistRecordsFromPage(Document doc)
    {

    ArrayList<MusicRecord> records = new ArrayList<MusicRecord>();
    
    Elements table_rows = doc.select("table.search_results tr");
    
    for (Element tr : table_rows) {
        
        Elements td_cells = tr.select("td");
        
        if (td_cells.size() == 6) {
        Element title_anchor = td_cells.get(0).select("a").first();
        String track_url       = title_anchor.attr("href");
        String title           = title_anchor.ownText();
        String instrumentation = td_cells.get(1).ownText();
        String composer        = td_cells.get(2).ownText();
        String year            = td_cells.get(3).ownText();
        
        Element audio_elem = td_cells.get(4).select("audio").first();
        if (audio_elem != null) {
            String audio_url = audio_elem.attr("src");
            
            MusicRecord mr = new MusicRecord(track_url,title,composer,year,audio_url);
            //System.out.println(mr.toJSONString());
            
            records.add(mr);
        }
        
        }                      
        
    }
    
    return records;
    }

    //private static void followArtistPages(String base_domain,String base_url, String artist_suffix)
    private static void followArtistPages(String base_domain,String starting_href)
    {
    String artist_works_url_str = base_domain+starting_href;

    try {
        boolean has_next_page = true;
        int page_number = 1;

        while (has_next_page) {
        System.out.println("Procesing Artist URL " + artist_works_url_str + ": Page " + page_number);

        Document doc = Jsoup.connect(artist_works_url_str).get();

        ArrayList<MusicRecord> music_records = extractArtistRecordsFromPage(doc);
        
        for (MusicRecord mr: music_records) {
            String audio_url = mr.getAudioURL();
            downloadURL(audio_url);

            saveJSONMetadata(mr,audio_url);

            //System.out.println(mr.toJSONString());

        }

        // <ul class="pagination">
        //  ...
        //  <li href="URL-TO-FOLLOW">Next</li>
        // </ul>

        Elements pagination_block = doc.select("ul.pagination > li > a");
        Element last_li_a = pagination_block.last();

        if (last_li_a != null) {
            String last_li_a_text = last_li_a.text();
            if (last_li_a_text.equals("Next >")) {
            artist_works_url_str = base_domain+last_li_a.attr("href");
            }
            else {
            has_next_page = false;
            }
        }
        else {
            has_next_page = false;
        }

        page_number++;
        }
    }
    catch (Exception e) {
        e.printStackTrace();
    }
    }

    
    /*
    public static void pageScrapeArtist(String artist_suffix)
    {

    // Example artist page at AMC
    //   http://www.australianmusiccentre.com.au/search?type=work&sort=alphaTitleSort&wfc[]=Roger+Dean

    String base_domain = "http://www.australianmusiccentre.com.au";
    String base_url = "/search?type=work&sort=alphaTitleSort";
    base_url += "&wfc[]=";

    followArtistPages(base_domain,base_url+"Roger+Dean");
    }
    */


    public static void main(String[] args)
    {

    // Represented artists at AMC
    //   http://www.australianmusiccentre.com.au/artists

    String base_domain = "http://www.australianmusiccentre.com.au";
    String base_url = "/artists";
    
    String represented_artists_url_str = base_domain+base_url;

    try {
        Document doc = Jsoup.connect(represented_artists_url_str).get();

        Elements artist_paras = doc.select("p.artist_home_name");
        
        for (Element ap : artist_paras) {
        
        Element artist_a = ap.select("a").first();
        if (artist_a != null) {
            String artist_about_url_str = base_domain + artist_a.attr("href");
            
            
            // Now process about this artist page to get to the sequence of pages
            Document about_doc = Jsoup.connect(artist_about_url_str).get();

            Element browse_by_artist = about_doc.select("#content > div.inner > ul > li > a").first();
            if (browse_by_artist != null) {

            String browse_works_href= browse_by_artist.attr("href");
            
            followArtistPages(base_domain,browse_works_href);
            }
        }
        }
    }
    catch (Exception e) {
        e.printStackTrace();
    }
    
    }
}