source: other-projects/ir-harvest/trunk/oai-download/org/waikato/harvest/OAIBase2IdList.java@ 31698

package org.waikato.harvest;

import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;


import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Reader;
import java.io.StringReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Set;

import org.waikato.harvest.OAIListSAXHandler;

public class OAIBase2IdList
{

    public static InputSource GetOAIListPage(String list_url, String list_args)
    {
    InputSource isource = null;
    
    try {
        HttpURLConnection httpcon = (HttpURLConnection) ((new URL(list_url).openConnection()));
        httpcon.setDoOutput(true);
        //httpcon.setRequestProperty("Content-Type", "application/json");
        //httpcon.setRequestProperty("Accept", "application/json");
        httpcon.setRequestMethod("GET");
        httpcon.connect();
        
        byte[] outputBytes = list_args.getBytes("UTF-8");
        OutputStream os = httpcon.getOutputStream();
        os.write(outputBytes);
        os.close();
        
        // Read response
        InputStream istream = httpcon.getInputStream();
        Reader isreader = new InputStreamReader(istream,"UTF-8");
        BufferedReader breader = new BufferedReader(isreader);
        isource = new InputSource(breader);
        isource.setEncoding("UTF-8");

        
    }
    catch (Exception e) {
        e.printStackTrace();
    }

    return isource;
    }
    
    public static void main( String[] args )
    {
    
    String[] oai_base_urls = new String[]
    {
        "http://nrl.northumbria.ac.uk/cgi/oai2",
        "http://discovery.ucl.ac.uk/cgi/oai2",
        "http://etheses.whiterose.ac.uk/cgi/oai2",
        "http://theses.gla.ac.uk/cgi/oai2"
    };

    for (String oai_base_url: oai_base_urls) {
        try {
        System.out.println("base url = " + oai_base_url);
        InputSource is = GetOAIListPage(oai_base_url, "verb=ListIdentifiers&metadataPrefix=oai_dc");
            
        SAXParserFactory factory = SAXParserFactory.newInstance();
        SAXParser saxParser = factory.newSAXParser();
        
        OAIListSAXHandler handler = new OAIListSAXHandler();
                
        saxParser.parse(is, handler);
        //is.close();
        
        handler.printIdentifiers();
        String resumptionToken = handler.getResumptionToken();
        
        
        }
        catch (Exception e) {
        e.printStackTrace();
        }

    }

    }
}