Changeset 18359 for gli

Show
Ignore:
Timestamp:
12.01.2009 11:18:36 (11 years ago)
Author:
kjdon
Message:

updated the rtl-gli branch with files from trunk. Result of a merge 14807:18318

Location:
gli/branches/rtl-gli/src/org/greenstone/gatherer/download
Files:
2 modified

Legend:

Unmodified
Added
Removed
  • gli/branches/rtl-gli/src/org/greenstone/gatherer/download/DownloadJob.java

    r13594 r18359  
    7676 
    7777    private String download_url = ""; 
    78  
     78     
    7979    //    private String current_url; 
    8080    //    private String destination; 
     
    9696    public static int DEFINED_MAX  = 1; 
    9797    public static int UNDEFINED_MAX  = 2; 
    98  
     98     
     99    // To prematurely terminate wget, we will need to use sockets and find a free port. 
     100    // We will look at a limited range of ports. This range will be reused (circular buffer) 
     101    private static final int PORT_BASE = 50000; 
     102    private static final int PORT_BLOCK_SIZE = 100;    
     103    private static int nextFreePort = PORT_BASE;  // Keep track what port numbers we have checked for availability 
     104    int port;                                     // package access. The socket port number this instance of DownloadJob will use 
     105     
    99106    private String mode = null; 
    100107    
    101108    private String proxy_url; 
    102  
     109     
    103110    /** 
    104111     */ 
     
    169176    // The stop_start_button is used to alternately start or stop the 
    170177    // job. If the current state of the job is paused then this 
    171     // restart is logically equivelent to a resume. 
     178    // restart is logically equivalent to a resume. 
    172179    if(event.getSource() == progress.stop_start_button) { 
    173180        previous_state = state; 
     
    189196    } 
    190197 
     198    /** Given a portnumber to check, returns true if it is available 
     199     * (if nothing's listening there already). */ 
     200    public static boolean isPortAvailable(int portnum) { 
     201    Socket tmpSocket = null; 
     202    try { 
     203        tmpSocket = new Socket("localhost", portnum); 
     204        tmpSocket.close(); 
     205        return false; 
     206 
     207    } catch(ConnectException ex){ 
     208        // "Signals that an error occurred while attempting to connect a socket  
     209        // to a remote address and port. Typically, the connection was refused 
     210        // remotely (e.g., no process is listening on the remote address/port)." 
     211        System.err.println("Port " + portnum + " not yet in use."); 
     212        tmpSocket = null; 
     213        return true; 
     214 
     215    } catch(Exception ex) {  
     216        // includes BindException "Signals that an error occurred while attempting 
     217        // to bind a socket to a local address and port. Typically, the port is in 
     218        // use, or the requested local address could not be assigned." 
     219        tmpSocket = null; 
     220        return false; 
     221    } 
     222    } 
     223 
     224    /** Circular buffer. Modifies the value of nextFreePort (the buffer index). */ 
     225    private void incrementNextFreePort() { 
     226    int offset = nextFreePort - PORT_BASE; 
     227    offset = (offset + 1) % PORT_BLOCK_SIZE; 
     228    nextFreePort = PORT_BASE + offset; 
     229    }  
    191230 
    192231    public void callDownload() { 
    193      
     232 
    194233    ArrayList command_list = new ArrayList(); 
    195234        if (Utility.isWindows()) { 
     
    202241    command_list.add("-cache_dir"); 
    203242    command_list.add(Gatherer.getGLIUserCacheDirectoryPath()); 
     243    // For the purposes of prematurely terminating wget from GLI (which creates a socket 
     244    // as a communication channel between GLI and Perl), it is important to tell the script 
     245    // that we're running as GLI. Because when running from the command prompt, it should 
     246    // not create this socket and do the related processing. 
     247    command_list.add("-gli"); 
    204248     
    205249    ArrayList all_arg = download.getArguments(true,false); 
     
    252296        } 
    253297        } 
    254   
    255298        //System.out.println(newcmd);        
    256299 
    257         InputStreamReader isr = new InputStreamReader(prcs.getErrorStream()); 
    258         BufferedReader br = new BufferedReader(isr); 
    259         // Capture the standard error stream and seach for two particular occurances. 
     300        // Can use the following if debugging WgetDownload.pm - Reads debug stmts from the perl process' STDIN stream 
     301        //(new PerlReaderThread(prcs)).start(); 
     302 
     303        InputStream is = prcs.getInputStream(); 
     304        BufferedReader reader = new BufferedReader(new InputStreamReader(is)); 
     305 
     306        // To be able to stop Wget, we use sockets to communicate with the perl process that launched wget 
     307        if (mode.equals("Web") || mode.equals("MediaWiki")) { // wget download modes other than OAI 
     308         
     309        // Need to find an available (unused) port within the range we're looking for to pass it 
     310        // the Perl child process, so that it may set up a listening ServerSocket at that port number 
     311        try { 
     312            boolean foundFreePort = false; 
     313            for(int i = 0; i < PORT_BLOCK_SIZE; i++) { 
     314 
     315            if(isPortAvailable(nextFreePort)) { 
     316                foundFreePort = true; 
     317                break; 
     318                 
     319            } else { 
     320                incrementNextFreePort(); 
     321            } 
     322            } 
     323             
     324            if(foundFreePort) { 
     325            // Free port number currently found becomes the port number of the socket that this 
     326            // DownloadJob instance will be connecting to when the user wants to prematurely stop Wget. 
     327            this.port = nextFreePort; 
     328            incrementNextFreePort(); 
     329             
     330            } else { 
     331            throw new Exception("Cannot find an available port in the range "  
     332                        + PORT_BASE + "-" + (PORT_BASE+PORT_BLOCK_SIZE) 
     333                        + "\nwhich is necessary for forcibly terminating wget."); 
     334            } 
     335 
     336            // Communicate the chosen port for this DownloadJob instance to the perl process, so 
     337            // that it can set up a ServerSocket at that port to listen for any signal to terminate wget 
     338            OutputStream os = prcs.getOutputStream(); 
     339            String p = ""+this.port+"\n"; 
     340            System.err.println("Portnumber found: " + p); 
     341 
     342            os.write(p.getBytes()); 
     343            os.close(); 
     344           
     345        } catch(Exception ex) { 
     346            System.err.println("Sent available portnumber " + this.port + " to process' outputstream.\nBut got exception: " + ex); 
     347        } 
     348        } 
     349         
     350        BufferedReader br = new BufferedReader(new InputStreamReader(prcs.getErrorStream())); 
     351        // Capture the standard error stream and search for two particular occurrences. 
    260352        String line=""; 
    261353        boolean ignore_for_robots = false; 
    262354        int max_download = DownloadJob.UNKNOWN_MAX; 
    263  
    264  
     355        
    265356        while ((line = br.readLine()) != null && !line.trim().equals("<<Finished>>") && state != STOPPED) { 
    266  
    267357        if ( max_download == DownloadJob.UNKNOWN_MAX) { 
    268358            if(line.lastIndexOf("<<Defined Maximum>>") != -1) { 
     
    355445        } 
    356446        } 
     447 
    357448        if(state == STOPPED) { 
    358         isr.close(); 
    359         prcs.destroy(); // This doesn't always work, but it's worth a try 
     449        boolean terminatePerlScript = true; 
     450         
     451        // When GLI is working with wget-based download modes other than OAI (MediaWiki and Web  
     452        // download) and the STOP button has been pressed, wget needs to be prematurely terminated. 
     453        // Only wget download modes Web and MediaWiki require the use of sockets to communicate 
     454        // with the perl script in order to get wget to terminate. Other download modes, including 
     455        // wgetdownload mode OAI, can terminate in the traditional manner: close process inputstream  
     456        // and kill perl process. OAI launches many wgets. So that when the perl process is terminated, 
     457        // the currently running wget will finish off but other wgets are no longer launched.  
     458        if(prcs != null && (mode.equals("Web") || mode.equals("MediaWiki"))) {  
     459 
     460            // create a socket to the perl child process and communicate the STOP message 
     461            Socket clientSocket = null; 
     462            if(clientSocket == null) { 
     463            try { 
     464                clientSocket = new Socket("localhost", this.port); // connect to the port chosen for this DownloadJob instance 
     465                 
     466                BufferedReader clientReader = new BufferedReader(new InputStreamReader(clientSocket.getInputStream())); 
     467                String response = clientReader.readLine(); // see if we've been connected 
     468                System.err.println("Communicating with perl download script on port " + this.port  
     469                           + "\nGot response from perl: " + response);  
     470                 
     471                // Send the STOP signal 
     472                OutputStream os = clientSocket.getOutputStream(); 
     473                String message = "<<STOP>>\n"; 
     474                os.write(message.getBytes()); 
     475                response = clientReader.readLine(); // see whether the stop signal has been received 
     476                System.err.println("GLI sent STOP signal to perl to terminate wget."  
     477                           + "\nGot response from perl: " + response); 
     478 
     479                response = clientReader.readLine(); // see whether the perl script is ready to be terminated 
     480                System.err.println("Got another response from perl: " + response); 
     481                os.close(); 
     482                 
     483                clientReader.close(); 
     484                clientSocket.close(); // close the clientSocket (the Perl end will close the server socket that Perl opened) 
     485                clientReader = null; 
     486                clientSocket = null; 
     487                 
     488                if(response == null) { 
     489                terminatePerlScript = false; 
     490                } 
     491            } catch(IOException ex) { 
     492                System.err.println("Tried to communicate through client socket - port " + this.port + ", but got exception: " + ex); 
     493            } catch(Exception ex) { 
     494                System.err.println("Tried to open client socket, but got exception: " + ex); 
     495            } 
     496            } 
     497        } 
     498         
     499        //prcs.getInputStream().close(); 
     500        prcs.getErrorStream().close(); 
     501        br.close(); 
     502        br = null; 
     503        if(terminatePerlScript) { 
     504            prcs.destroy(); // This doesn't always work, but it's worth a try        
     505            prcs = null; 
     506        } 
     507 
     508        // Notify the DownloadScrollPane which is waiting on this job to complete that we are ready  
     509        synchronized(this) { 
     510            this.notify(); 
     511        } 
    360512        } 
    361         
    362513    } 
    363514    catch (Exception ioe) { 
     
    372523        previous_state = state; 
    373524        state = DownloadJob.COMPLETE; 
    374          
    375525    } 
    376526    // refresh the workspace tree 
    377527    Gatherer.g_man.refreshWorkspaceTree(WorkspaceTree.DOWNLOADED_FILES_CHANGED); 
    378         
    379528    } 
    380529 
     
    464613    progress.updateProgress(current, expected); 
    465614    } 
     615 
     616     
     617    // Inner thread class that reads from process downloadfrom.pl's errorstream 
     618    private class PerlReaderThread extends Thread { 
     619    Process prcs = null; 
     620 
     621    public PerlReaderThread(Process proc) { 
     622        this.prcs = proc; 
     623    } 
     624 
     625    public void run() { 
     626        try { 
     627        if(prcs != null) { 
     628            String message = null; 
     629            BufferedReader eReader = new BufferedReader(new InputStreamReader(prcs.getInputStream())); 
     630            while(prcs != null && (message = eReader.readLine()) != null) { 
     631            if(!message.equals("\n")) { 
     632                System.err.println("**** Perl STDOUT: " + message); 
     633            } 
     634            } 
     635             
     636            if(prcs != null && eReader != null) { 
     637            eReader.close(); 
     638            eReader = null; 
     639            System.err.println("**** Perl ENDed."); 
     640            } 
     641        } 
     642        } catch(Exception e) { 
     643        System.err.println("Thread - caught exception: " + e); 
     644        } 
     645    } 
     646    } 
    466647} 
  • gli/branches/rtl-gli/src/org/greenstone/gatherer/download/DownloadScrollPane.java

    r12529 r18359  
    5252    extends Thread { 
    5353     
    54     /** <i>true</i> if there is a task currently being carried out, <i>false</i> otherwise. */ 
    55     private boolean busy = false; 
    5654    /** <i>true</i> if verbose debug messages should be displayed, <i>false</i> otherwise. */ 
    5755    private boolean debug = false; 
     
    6967    private Vector job_queue; 
    7068    static final private boolean simple = true; 
    71      
     69 
     70         
    7271    public DownloadScrollPane() { 
    7372    job = null; 
     
    8180    public void deleteDownloadJob(DownloadJob delete_me) { 
    8281    if (delete_me == job) { 
    83         while(busy) { 
    84         } 
    85         job = null; 
    86     } 
     82        try { 
     83        // Wait for the job to finish cleaning up, before we can continue cleaning up here. 
     84        // But sometimes the job has completed (terminated naturally) and in that case there's 
     85        // nothing to wait for. 
     86        synchronized(delete_me) { 
     87            if (!delete_me.hasSignalledStop()) { // don't wait if DownloadJob.COMPLETED  
     88            delete_me.wait(); 
     89            } 
     90        } 
     91        } catch (Exception e) { 
     92        e.printStackTrace(); 
     93        }        
     94    } 
     95    // Close button pressed, get rid of this download's dedicated pane 
     96    finishedDownloadJob(delete_me, true); 
     97    } 
     98 
     99    /** To be called when a download job has terminated naturally or was prematurely stopped 
     100     *  via the close button.  
     101     *  Gets rid of this download's pane with buttons and progress bar if prematurely stopped. */ 
     102    protected void finishedDownloadJob(DownloadJob delete_me, boolean removeDisplay) { 
    87103    if (delete_me.hasSignalledStop()) { 
    88         list_pane.remove(delete_me.getProgressBar()); 
     104        if(removeDisplay) { 
     105        list_pane.remove(delete_me.getProgressBar()); 
     106        list_pane.remove(filler_pane); 
     107        } 
    89108        job_queue.remove(delete_me); 
    90         list_pane.remove(filler_pane); 
    91109        if(job_queue.size() > 0) { 
    92110        Dimension progress_bar_size = delete_me.getProgressBar().getPreferredSize(); 
     
    131149 
    132150    DebugStream.println("About to create a new job"); 
    133  
    134  
     151     
    135152        DownloadJob new_job = new DownloadJob(download, Configuration.proxy_pass, Configuration.proxy_user, this, mode, proxy_url); 
    136153    // Tell it to run as soon as possible 
    137  
    138        
    139154 
    140155    new_job.setState(DownloadJob.RUNNING); 
     
    165180    //list_pane.setAlignmentX(Component.LEFT_ALIGNMENT); 
    166181    list_pane.updateUI(); 
    167     new_job = null; 
     182    new_job = null;         //job = (DownloadJob) job_queue.get(index); 
     183 
    168184    synchronized(this) { 
    169185        notify(); // Just incase its sleeping. 
     
    189205    while(true) { 
    190206        if(job_queue.size() > 0) { 
    191         int index = 0; 
    192         while(job_queue.size() > 0 && index < job_queue.size()) { 
    193             job = (DownloadJob) job_queue.get(index); 
     207        while(!job_queue.isEmpty()) {  
     208            job = (DownloadJob) job_queue.firstElement();  
     209             
    194210            if(job.getState() == DownloadJob.RUNNING) { 
    195             DebugStream.println("DownloadJob " + job.toString() + " Begun."); 
    196             busy = true; 
     211            DownloadJob delete_me = job; 
     212            String jobDisplayString = job.toString(); 
     213            DebugStream.println("DownloadJob " + jobDisplayString + " Begun."); 
     214            System.err.println("DownloadJob " + job.port + " " + job.toString() + " Begun."); 
    197215            job.callDownload(); 
    198             busy = false; 
    199             DebugStream.println("DownloadJob " + job.toString() + " complete."); 
     216            // Job is done. Ended naturally, don't get rid of this download's separate display panel 
     217            finishedDownloadJob(delete_me, false);  
     218            System.err.println("DownloadJob " + jobDisplayString + " complete."); 
     219            DebugStream.println("DownloadJob " + jobDisplayString + " complete."); // by this point job is null! 
    200220            job = null; 
    201             } 
    202             index++; 
     221            delete_me = null; 
     222            } 
    203223        } 
    204224        try {