root/trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GoogleSearch.java.tmp @ 11228

Revision 11228, 7.2 KB (checked in by kjdon, 14 years ago)

minor change

  • Property svn:keywords set to Author Date Id Revision
Line 
1package org.greenstone.gsdl3.service;
2
3// Greenstone classes
4import org.greenstone.gsdl3.util.*;
5
6//Google Web Services API classes
7//import com.google.soap.search.GoogleSearch;
8import com.google.soap.search.GoogleSearchFault;
9import com.google.soap.search.GoogleSearchResult;
10import com.google.soap.search.GoogleSearchResultElement;
11import com.google.soap.search.GoogleSearchDirectoryCategory;
12
13// XML classes
14import org.w3c.dom.Element;
15import org.w3c.dom.Document;
16import org.w3c.dom.NodeList;
17
18//Java classes
19import java.util.ArrayList;
20import java.util.HashMap;
21import java.io.File;
22import java.io.InputStream;
23import java.io.BufferedReader;
24import java.io.InputStreamReader;
25import java.io.IOException;
26import java.net.HttpURLConnection;
27import java.net.URLConnection;
28import java.net.URL;
29import java.net.Authenticator;
30import java.net.PasswordAuthentication;
31import java.net.MalformedURLException;
32import java.lang.Object;
33
34
35/**
36 *
37 * @author <a href="mailto:chi@cs.waikato.ac.nz">Chi-Yu Huang</a>
38 *
39 */
40
41public class GoogleSearch
42    extends AbstractSearch {
43   
44    //Parameters connect to Proxy Server
45    private boolean using_proxy = false;
46    private String proxy_host = null;
47    private int proxy_port;
48    private String proxy_user = null;
49    private char [] proxy_passwd = null;
50
51    // google key
52    private String client_key = null;
53   
54    public GoogleSearch()
55    {
56    }
57       
58    //Configure GoogleSearch Service
59    public boolean configure(Element info, Element extra_info)
60    {
61    if (!super.configure(info, extra_info)) {
62        return false;
63    }
64    System.err.println("Configuring GoogleSearch");
65    Element server_elem = (Element)GSXML.getChildByTagName(info, "googleServer");
66    if (server_elem == null) {
67        System.err.println("GoogleSearch.configure error: no googleServer element found");
68        return false;
69    }
70   
71    client_key = server_elem.getAttribute("key");
72    if (client_key.equals("")) {
73        System.err.println("GoogleSearch.configure error: no client_key for the googleServer element");
74        return false;
75    }
76   
77    does_paging = true;
78    does_chunking = false;
79
80    // are we behind a proxy??
81    // all the details should have been set up by the Message Router
82    proxy_host = System.getProperty("http.proxyHost");
83    if (proxy_host != null && !proxy_host.equals("")) {
84        using_proxy = true;
85        try {
86        proxy_port = Integer.parseInt(System.getProperty("http.proxyPort").trim());
87        } catch (Exception e) {
88        System.err.println("GoogleSearch.configure error: couldn't get proxy port, defaulting to 80");
89        proxy_port = 80;
90        }
91        PasswordAuthentication pa = Authenticator.requestPasswordAuthentication(proxy_host, null, proxy_port, "http", "", null);
92        proxy_user = pa.getUserName();
93        proxy_passwd = pa.getPassword();
94    }
95    return true;
96    }
97   
98    /** Process a text query - implemented by concrete subclasses */
99    protected Element processTextQuery(Element request) {
100   
101    //Connect to Google Web API service
102    com.google.soap.search.GoogleSearch search = new com.google.soap.search.GoogleSearch();
103
104    // Set mandatory attributes
105    search.setKey(client_key);
106
107    // proxy??
108    if (using_proxy) {
109        search.setProxyHost(proxy_host);
110        search.setProxyPort(proxy_port);
111        search.setProxyUserName(proxy_user);
112        search.setProxyPassword(new String(proxy_passwd));
113    }
114
115    //set optional attributes
116    search.setSafeSearch(true);
117   
118    // Create a new (empty) result message
119    Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
120    result.setAttribute(GSXML.FROM_ATT, TEXT_QUERY_SERVICE);
121    result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
122    Element doc_node_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
123    result.appendChild(doc_node_list);
124    Element query_metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
125    result.appendChild(query_metadata_list);
126
127    Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
128    if (param_list == null) {
129        System.err.println("GoogleSearch.processTextQuery Error:: TextQuery request had no paramList.");
130        return result;  // Return the empty result
131    }
132
133    // Process the request parameters
134    HashMap params = GSXML.extractParams(param_list, false);
135
136    // Make sure a query has been specified
137    String query = (String) params.get(QUERY_PARAM);
138    if (query == null || query.equals("")) {
139        System.err.println("GoogleSearch.processTextQuery Error: TextQuery request had no query string.");
140        return result;  // Return the empty result
141    }
142        // tidy whitespace
143    query = query.replaceAll("\\s+", "+");
144   
145    search.setQueryString(query);
146
147    //Check hits_per_page
148    int hits_per_page;
149    try {
150        hits_per_page = Integer.parseInt(((String)params.get(HITS_PER_PAGE_PARAM)).trim());
151    } catch (Exception e) {
152        System.err.println("GoogleSearch.processTextQuery error: couldn't get hits per page param, defaulting to 10");
153        hits_per_page = 10;
154    }
155    //Check the start_page number
156    int start_page;
157    try {
158        start_page = Integer.parseInt(((String) params.get(START_PAGE_PARAM)).trim());
159    } catch (Exception e) {
160        System.err.println("GoogleSearch.processTextQuery error: couldn't get start page param, defaulting to 1");
161        start_page = 1;
162    }
163   
164    //Invoke Actual Search
165   
166    // Google only allows 10 hits per request
167    int loop = hits_per_page/10;
168    int remainder = hits_per_page%10;
169    int google_start_page = (start_page-1)*hits_per_page;
170    int pages_per_loop;
171    for (int j=0; j < loop; j++){
172        if (j < (loop-1) || remainder == 0) {
173        pages_per_loop = 10;
174        } else {
175        pages_per_loop = remainder;
176        }
177        search.setMaxResults(pages_per_loop);
178        search.setStartResult(google_start_page);
179        google_start_page = google_start_page + pages_per_loop;
180        GoogleSearchResult google_result;
181        try{
182        google_result = search.doSearch();
183        } catch (GoogleSearchFault ex) {
184        System.err.println("GoogleSearch.processTextQuery error: the call to the Google Web APIs failed:" + ex.toString());
185        // add the error to the result
186        return result;
187        }
188        if (j==0) {
189       
190        //Total amount of documents Google Search returned
191        // only need to do this on the first loop
192       
193        long numdocs_matched = google_result.getEstimatedTotalResultsCount();       
194        GSXML.addMetadata(this.doc, query_metadata_list, "numDocsMatched", ""+numdocs_matched);
195       
196        }
197        GoogleSearchResultElement[] details = google_result.getResultElements();
198        for (int i=0; i<details.length; i++){
199        Element doc_node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
200        doc_node_list.appendChild(doc_node);
201       
202        Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
203        doc_node.appendChild(metadata_list);
204       
205        String google_url = details[i].getURL(); 
206        String google_title = details[i].getTitle();
207        String google_snippet = details[i].getSnippet();
208        if (google_url !=null) {
209            GSXML.addMetadata(this.doc, metadata_list, "URL", google_url);
210        }
211        if (google_title != null) {
212            GSXML.addMetadata(this.doc, metadata_list, "Title", google_title);
213        }
214        if (google_snippet != null) {
215            GSXML.addMetadata(this.doc, metadata_list, "Snippet", google_snippet);
216        }
217        }
218    } // for each loop
219    return result;
220    }
221   
222    protected void getIndexData(ArrayList index_ids, ArrayList index_names,String lang){
223        index_ids.add("idx");
224    index_names.add("Google main index");
225    }
226   
227}
Note: See TracBrowser for help on using the browser.