source: gs3-extensions/hathitrust-downloadfrom/trunk/htc-catalog-search.pl@ 26436

#!/usr/bin/perl -w

use strict;
use warnings;

use LWP;

use OAuth::Lite::Consumer;
use OAuth::Lite::AuthMethod;

use WWW::Mechanize;

# use CGI;


sub data_api
{
    my ($doc_id) = @_;

    #my $access_key = 'PUBLIC_OAUTH_CONSUMER_KEY';
    #my $secret_key = 'PUBLIC_OAUTH_CONSUMER_SECRET';

    my $access_key = '7e6ee38bae';                   # PUBLIC_OAUTH_CONSUMER_KEY
    my $secret_key = 'e0429c0394385486249b4a230702'; # PUBLIC_OAUTH_CONSUMER_SECRET

    #my $request_url = 'http://babel.hathitrust.org/cgi/htd/dapiserver';
    #my $request_url = "http://babel.hathitrust.org/cgi/htd/meta/mdp.39015019203879";
    my $request_url = "http://babel.hathitrust.org/cgi/htd/pagemeta/mdp.39015000000128/12";


    my $consumer = OAuth::Lite::Consumer->new( 'consumer_key' => $access_key,
                           'consumer_secret' => $secret_key,
                           'auth_method' => OAuth::Lite::AuthMethod::URL_QUERY );

    my $response = $consumer->request( 'method' => 'GET',
                       'url' => $request_url,
 #                     'params' => { 'hello' => 'world' } 
    );

#    print CGI::header();

#    print "<p><b>[CLIENT] sent this URL to server:</b><br/>";
#    print $consumer->oauth_request->uri;
    
#    print "<p><b>[CLIENT] received this HTTP response from server:</b><br/>";
#    print $response->status_line;
    
    if ($response->is_success) {
#   print "<br/><b>[CLIENT] received this content response from server:</b><blockquote>" .
#       $response->content . "</blockquote>";

    print "Recieved content:\n";
    print "------\n";

    print $response->content()
    }
    else {
    print STDERR "**** Failed to retrieval any content from URL:\n";
    print STDERR "         ", $consumer->oauth_request->uri, "\n";
    print STDERR "**** Status: ", print $response->status_line, "\n";
    
    }


##    print STDERR "*****\n ", $consumer->oauth_request->uri, "\n";
}


sub bibliographic_api
{
    my ($catalog_id) = @_;
    
    my $catalog_json = "$catalog_id.json";
    my $base_url = "http://catalog.hathitrust.org/api/volumes/full/recordnumber";
    my $url = "$base_url/$catalog_json";

    my $ua = LWP::UserAgent->new();
#    $ua->agent("Greenstone DL Ingest");

    # make request
    my $request = HTTP::Request->new(GET => $url);

    # get response
    my $response = $ua->request($request);

    if ($response->is_success()) {

    my $content_type = $response->content_type();
    
    my $content = $response->content();

    my $group_by_dir = "output";

    if (!-d $group_by_dir) {
        print "Creating '$group_by_dir'\n";
        mkdir($group_by_dir);
    }

    my @group_by = ($catalog_id =~ m/\d{1,2}/g);

    while (my $next_subdir = shift @group_by) {
        $group_by_dir .= "/$next_subdir";
        if (!-d $group_by_dir) {
        mkdir($group_by_dir);
        }

        last if (scalar(@group_by)==1);     
    }

    my $ofilename = "$group_by_dir/$catalog_json";
    if (!-e $ofilename) {
        if (open(JOUT,">$ofilename")) {
        
        print JOUT $content;
        print JOUT "\n";
        close(JOUT);
        }
        else {
        print STDERR "Error: Failed to open $ofilename\n";
        print STDERR "!$\n";
        }
    }
    else {
        print STDOUT "$ofilename already exists.  Skipping.\n";
    }
    }
    else 
    {
    print STDERR "Error: Failed to retrieve $url\n";
    print STDERR "-----\n";
    print STDERR "Status line: ", $response->status_line(), "\n";
    print STDERR "    ", $response->content(),"\n";
    }
}



sub main
{

    my ($argv_ref) = @_;

    my $query=join("+",@$argv_ref) || "zealand";
    
    my $base_url = "http://catalog.hathitrust.org/Search/Home?checkspelling=true&type=all&submit=&type=all&sethtftonly=true";
    my $url = $base_url . "&lookfor=" . $query;

    my $mech = WWW::Mechanize->new();
    $mech->get($url);
    
    my $next_link = $mech->find_link( text_regex => qr/^Next\s+/);
    
    my $count=0;
    
    while (defined($next_link)) {


    my $catalog_links    = $mech->find_all_links(text_regex => qr/^Catalog Record\s*/);
#    my $full_links       = $mech->find_all_links(text_regex => qr/^Full view\s*$/,
#                        url_regex => qr/hdl\.handle\.net/);
#    my $restricted_links = $mech->find_all_links(text_regex => qr/^Limited \(search-only\)/,  
#                        url_regex => qr/hdl\.handle\.net/);

    my $num_catalog_links    = scalar(@$catalog_links);
#    my $num_full_links       = scalar(@$full_links);
#    my $num_restricted_links = scalar(@$restricted_links);

#    print "+++++ num cat links $num_catalog_links: num full = $num_full_links, num restricted = $num_restricted_links\n";

    
    foreach my $cat_link (@$catalog_links) {
        my $cat_url = $cat_link->url();
        my ($cat_id) = ($cat_url =~ m/\/([^\/]*)$/);
        print "cat id = $cat_id\n";
        bibliographic_api($cat_id);
    }


    $url = $next_link->url();
    
    $mech->get($url);
    $next_link = $mech->find_link( text_regex => qr/^Next\s+/);

    $count++;
    
##  last if ($count==1);

    print "Away to Process link: $url\n"; 

    }
    
}


main(\@ARGV);