Changeset 31663


Ignore:
Timestamp:
05/08/17 18:15:32 (4 years ago)
Author:
ak19
Message:

Bringing the GS3 SafeProcess.java up to speed with the GLI version. Now adding the 2 jna jar files that SafeProcess uses on Windows upon process.destroy(), to terminate further subprocesses that may have got launched.

Location:
main/trunk/greenstone3
Files:
2 added
2 edited

Legend:

Unmodified
Added
Removed
  • main/trunk/greenstone3/src/java/org/greenstone/util/SafeProcess.java

    r31640 r31663  
    1111import java.io.OutputStreamWriter;
    1212import java.util.Arrays;
     13import java.util.Scanner;
     14import java.util.Stack;
     15
     16import com.sun.jna.*;
     17import com.sun.jna.platform.win32.Kernel32;
     18import com.sun.jna.platform.win32.WinNT;
     19
     20import java.lang.reflect.Field;
     21//import java.lang.reflect.Method;
    1322
    1423import org.apache.log4j.*;
     
    2029// to avoid blocking problems that can arise from a Process' input and output streams.
    2130
     31// On Windows, Perl could launch processes as proper ProcessTrees: http://search.cpan.org/~gsar/libwin32-0.191/
     32// Then killing the root process will kill child processes naturally.
     33
    2234public class SafeProcess {
    23     //public static int DEBUG = 0;
     35    public static int DEBUG = 0;
    2436
    2537    public static final int STDERR = 0;
    2638    public static final int STDOUT = 1;
    2739    public static final int STDIN = 2;
    28 
     40    // can't make this variable final and init in a static block, because it needs to use other SafeProcess static methods which rely on this in turn:
     41    public static String WIN_KILL_CMD;
     42       
     43   
    2944    // charset for reading process stderr and stdout streams
    3045    //public static final String UTF8 = "UTF-8";   
     
    3954    private String inputStr = null;
    4055    private Process process = null;
    41 
     56    private boolean forciblyTerminateProcess = false;
     57   
    4258    // output from running SafeProcess.runProcess()
    4359    private String outputStr = "";
     
    102118    }
    103119
     120    // In future, think of changing the method doRuntimeExec() over to using ProcessBuilder
     121    // instead of Runtime.exec(). ProcessBuilder seems to have been introduced from Java 5.
     122    // https://docs.oracle.com/javase/7/docs/api/java/lang/ProcessBuilder.html
     123    // https://zeroturnaround.com/rebellabs/how-to-deal-with-subprocesses-in-java/
     124    // which suggests using Apache Common Exec to launch processes and says what will be forthcoming in Java 9
     125   
    104126    private Process doRuntimeExec() throws IOException {
    105127    Process prcs = null;
     
    113135       
    114136        // http://stackoverflow.com/questions/5283444/convert-array-of-strings-into-a-string-in-java
    115         log("SafeProcess running: " + Arrays.toString(command_args));
     137        //log("SafeProcess running:" + Arrays.toString(command_args));
     138        StringBuffer cmdDisplay = new StringBuffer();
     139        for(int i = 0; i < command_args.length; i++) {
     140        cmdDisplay.append(" ").append(command_args[i]);
     141        }
     142        log("SafeProcess running: [" + cmdDisplay + "]");
     143        cmdDisplay = null; // let the GC have it   
     144       
    116145       
    117146        if(this.envp == null) {
     
    139168    throws IOException, InterruptedException
    140169    {
    141 
    142    
    143170    // kick off the stream gobblers
    144171    inputGobbler.start();
     
    176203        errorGobbler.interrupt();
    177204        outputGobbler.interrupt();
    178 
     205       
     206        // Since we have been cancelled (InterruptedException), or on any Exception, we need
     207        // to forcibly terminate process eventually after the finally code first waits for each worker thread
     208        // to die off. Don't set process=null until after we've forcibly terminated it if needs be.
     209        this.forciblyTerminateProcess = true;
     210       
    179211        // even after the interrupts, we want to proceed to calling join() on all the worker threads
    180212        // in order to wait for each of them to die before attempting to destroy the process if it
     
    204236        // set the variables that the code which created a SafeProcess object may want to inspect
    205237        this.outputStr = outputGobbler.getOutput();
    206         this.errorStr = errorGobbler.getOutput();
    207        
    208         // Since we didn't have an exception, process should have terminated now (waitFor blocks until then)
    209         // Set process to null so we don't forcibly terminate it below with process.destroy()
    210         this.process = null;       
     238        this.errorStr = errorGobbler.getOutput();       
    211239    }
    212240
     
    223251
    224252    // Run a very basic process: with no reading from or writing to the Process' iostreams,
    225     // this just execs the process and waits for it to return
     253    // this just execs the process and waits for it to return.
     254    // Don't call this method but the zero-argument runProcess() instead if your process will
     255    // output stuff to its stderr and stdout streams but you don't need to monitory these.
     256    // Because, as per a comment in GLI's GS3ServerThread.java,
     257    // in Java 6, it wil block if you don't handle a process' streams when the process is
     258    // outputting something. (Java 7+ won't block if you don't bother to handle the output streams)
    226259    public int runBasicProcess() {
    227260    try {
     261        this.forciblyTerminateProcess = true;
     262       
    228263        // 1. create the process
    229264        process = doRuntimeExec();
     
    231266        this.exitValue = process.waitFor();
    232267
    233        
    234     } catch(IOException ioe) {
     268        this.forciblyTerminateProcess = false;
     269    } catch(IOException ioe) {     
     270       
    235271        if(exceptionHandler != null) {
    236272        exceptionHandler.gotException(ioe);
     
    249285    } finally {
    250286
    251         if( process != null ) {
    252         process.destroy(); // see runProcess() below
    253         }
     287        if( this.forciblyTerminateProcess ) {
     288        destroyProcess(process); // see runProcess() below     
     289        }
     290        process = null;
     291        this.forciblyTerminateProcess = false; // reset
    254292    }
    255293    return this.exitValue;
     
    273311
    274312    try {
     313        this.forciblyTerminateProcess = false;
     314       
    275315        // 1. get the Process object
    276316        process = doRuntimeExec();
     
    291331        if(procErrHandler == null) {
    292332        errorGobbler // ReaderFromProcessOutputStream
    293             = new SafeProcess.InputStreamGobbler(process.getErrorStream(), splitStdErrorNewLines);
     333            = new SafeProcess.InputStreamGobbler(process.getErrorStream(), this.splitStdErrorNewLines);
    294334        } else {
    295335        errorGobbler
     
    300340        if(procOutHandler == null) {
    301341        outputGobbler
    302             = new SafeProcess.InputStreamGobbler(process.getInputStream(), splitStdOutputNewLines);
     342            = new SafeProcess.InputStreamGobbler(process.getInputStream(), this.splitStdOutputNewLines);
    303343        } else {
    304344        outputGobbler
     
    311351       
    312352    } catch(IOException ioe) {
     353        this.forciblyTerminateProcess = true;
     354
    313355        if(exceptionHandler != null) {
    314356        exceptionHandler.gotException(ioe);
     
    317359        }
    318360    } catch(InterruptedException ie) { // caused during any of the gobblers.join() calls, this is unexpected so print stack trace
    319        
     361        this.forciblyTerminateProcess = true;
     362       
    320363        if(exceptionHandler != null) {
    321364        exceptionHandler.gotException(ie);
     
    332375        //log("*** In finally of SafeProcess.runProcess(3 params): " + cmd);
    333376
    334         if( process != null ) {
     377        if( this.forciblyTerminateProcess ) {
    335378        log("*** Going to call process.destroy 2");
    336         process.destroy();
    337         process = null;
     379        destroyProcess(process);               
    338380        log("*** Have called process.destroy 2");
    339381        }
    340        
     382        process = null;
     383        this.forciblyTerminateProcess = false; // reset
    341384    }
    342385   
     
    351394
    352395    try {
     396        this.forciblyTerminateProcess = false;
     397       
    353398        // 1. get the Process object
    354399        process = doRuntimeExec();
     
    379424
    380425
    381             // 4. kick off the stream gobblers
     426        // 4. kick off the stream gobblers
    382427        this.exitValue = waitForWithStreams(inputGobbler, outputGobbler, errorGobbler);
    383428       
    384429    } catch(IOException ioe) {
     430        this.forciblyTerminateProcess = true;
     431       
    385432        if(exceptionHandler != null) {
    386433        exceptionHandler.gotException(ioe);
     
    389436        }
    390437    } catch(InterruptedException ie) { // caused during any of the gobblers.join() calls, this is unexpected so log it
    391        
     438        this.forciblyTerminateProcess = true;
     439       
    392440        if(exceptionHandler != null) {
    393441        exceptionHandler.gotException(ie);
     
    415463        //log("*** In finally of SafeProcess.runProcess(2 params): " + cmd);
    416464
    417         if( process != null ) {
     465        if( this.forciblyTerminateProcess ) {
    418466        log("*** Going to call process.destroy 1");
    419         process.destroy();
    420         process = null;
     467        destroyProcess(process);   
    421468        log("*** Have called process.destroy 1");
    422469        }
     470        process = null;
     471        this.forciblyTerminateProcess = false; //reset     
    423472    }
    424473   
     
    426475    }
    427476
    428 
     477/*
     478
     479 On Windows, p.destroy() terminates process p that Java launched,
     480 but does not terminate any processes that p may have launched. Presumably since they didn't form a proper process tree.
     481    https://social.msdn.microsoft.com/Forums/windowsdesktop/en-US/e3cb7532-87f6-4ae3-9d80-a3afc8b9d437/how-to-kill-a-process-tree-in-cc-on-windows-platform?forum=vclanguage
     482    https://msdn.microsoft.com/en-us/library/windows/desktop/ms684161(v=vs.85).aspx
     483
     484 Searching for: "forcibly terminate external process launched by Java on Windows"   
     485 Not possible: stackoverflow.com/questions/1835885/send-ctrl-c-to-process-open-by-java
     486 But can use taskkill or tskill or wmic commands to terminate a process by processID
     487 stackoverflow.com/questions/912889/how-to-send-interrupt-key-sequence-to-a-java-process
     488 Taskkill command can kill by Image Name, such as all running perl, e.g. taskkill /f /im perl.exe
     489 But what if we kill perl instances not launched by GS?
     490    /f Specifies to forcefully terminate the process(es). We need this flag switched on to kill childprocesses.
     491    /t Terminates the specified process and any child processes which were started by it.
     492            /t didn't work to terminate subprocesses. Maybe since the process wasn't launched as
     493            a properly constructed processtree.
     494    /im is the image name (the name of the program), see Image Name column in Win Task Manager.
     495   
     496 We don't want to kill all perl running processes.
     497 Another option is to use wmic, available since Windows XP, to kill a process based on its command
     498 which we sort of know (SafeProcess.command) and which can be seen in TaskManager under the
     499 "Command Line" column of the Processes tab.
     500    https://superuser.com/questions/52159/kill-a-process-with-a-specific-command-line-from-command-line
     501 The following works kill any Command Line that matches -site localsite lucene-jdbm-demo
     502    C:>wmic PATH win32_process Where "CommandLine like '%-site%localsite%%lucene-jdbm-demo%'" Call Terminate
     503 "WMIC Wildcard Search using 'like' and %"
     504    https://codeslammer.wordpress.com/2009/02/21/wmic-wildcard-search-using-like-and/
     505 However, we're not even guaranteed that every perl command GS launches will contain the collection name
     506 Nor do we want to kill all perl processes that GS launches with bin\windows\perl\bin\perl, though this works:
     507    wmic PATH win32_process Where "CommandLine like '%bin%windows%perl%bin%perl%'" Call Terminate
     508 The above could kill GS perl processes we don't intend to terminate, as they're not spawned by the particular
     509 Process we're trying to terminate from the root down.
     510   
     511 Solution: We can use taskkill or the longstanding tskill or wmic to kill a process by ID. Since we can
     512 kill an external process that SafeProcess launched OK, and only have trouble killing any child processes
     513 it launched, we need to know the pids of the child processes.
     514 
     515 We can use Windows' wmic to discover the childpids of a process whose id we know.
     516 And we can use JNA to get the process ID of the external process that SafeProcess launched.
     517 
     518 To find the processID of the process launched by SafeProcess,
     519 need to use Java Native Access (JNA) jars, available jna.jar and jna-platform.jar.
     520    http://stackoverflow.com/questions/4750470/how-to-get-pid-of-process-ive-just-started-within-java-program
     521    http://stackoverflow.com/questions/35842/how-can-a-java-program-get-its-own-process-id
     522    http://www.golesny.de/p/code/javagetpid
     523    https://github.com/java-native-access/jna/blob/master/www/GettingStarted.md
     524 We're using JNA v 4.1.0, https://mvnrepository.com/artifact/net.java.dev.jna/jna
     525 
     526 WMIC can show us a list of parent process id and process id of running processes, and then we can
     527 kill those child processes with a specific process id.
     528    https://superuser.com/questions/851692/track-which-program-launches-a-certain-process
     529    http://stackoverflow.com/questions/7486717/finding-parent-process-id-on-windows
     530 WMIC can get us the pids of all childprocesses launched by parent process denoted by parent pid.
     531 And vice versa:
     532    if you know the parent pid and want to know all the pids of the child processes spawned:
     533        wmic process where (parentprocessid=596) get processid
     534    if you know a child process id and want to know the parent's id:
     535        wmic process where (processid=180) get parentprocessid
     536 
     537 The above is the current solution.
     538 
     539 Eventually, instead of running a windows command to kill the process ourselves, consider changing over to use
     540    https://github.com/flapdoodle-oss/de.flapdoodle.embed.process/blob/master/src/main/java/de/flapdoodle/embed/process/runtime/Processes.java
     541 (works with Apache license, http://www.apache.org/licenses/LICENSE-2.0)
     542 This is a Java class that uses JNA to terminate processes. It also has the getProcessID() method.
     543 
     544 Linux ps equivalent on Windows is "tasklist", see
     545    http://stackoverflow.com/questions/4750470/how-to-get-pid-of-process-ive-just-started-within-java-program
     546
     547*/
     548
     549// http://stackoverflow.com/questions/4750470/how-to-get-pid-of-process-ive-just-started-within-java-program
     550// Uses Java Native Access, JNA
     551public static long getProcessID(Process p)
     552{
     553    long pid = -1;
     554    try {
     555    //for windows
     556    if (p.getClass().getName().equals("java.lang.Win32Process") ||
     557        p.getClass().getName().equals("java.lang.ProcessImpl"))
     558        {
     559        Field f = p.getClass().getDeclaredField("handle");
     560        f.setAccessible(true);             
     561        long handl = f.getLong(p);
     562        Kernel32 kernel = Kernel32.INSTANCE;
     563        WinNT.HANDLE hand = new WinNT.HANDLE();
     564        hand.setPointer(Pointer.createConstant(handl));
     565        pid = kernel.GetProcessId(hand);
     566        f.setAccessible(false);
     567        }
     568    //for unix based operating systems
     569    else if (p.getClass().getName().equals("java.lang.UNIXProcess"))
     570        {
     571        Field f = p.getClass().getDeclaredField("pid");
     572        f.setAccessible(true);
     573        pid = f.getLong(p);
     574        f.setAccessible(false);
     575        }
     576
     577    } catch(Exception ex) {
     578    log("SafeProcess.getProcessID(): Exception when attempting to get process ID for process " + ex.getMessage(), ex); 
     579    pid = -1;
     580    }
     581    return pid;
     582}
     583   
     584
     585// stackoverflow.com/questions/1835885/send-ctrl-c-to-process-open-by-java 
     586// (Taskkill command can kill all running perl. But what if we kill perl instances not launched by GS?)
     587// stackoverflow.com/questions/912889/how-to-send-interrupt-key-sequence-to-a-java-process
     588// Searching for: "forcibly terminate external process launched by Java on Windows"
     589static void killWinProcessWithID(long processID) {
     590   
     591    String cmd = SafeProcess.getWinProcessKillCmd(processID);
     592    if (cmd == null) return;
     593   
     594    try {       
     595    log("\tAttempting to terminate Win subprocess with pid: " + processID);
     596    SafeProcess proc = new SafeProcess(cmd);           
     597    int exitValue = proc.runProcess(); // no IOstreams for Taskkill, but for "wmic process pid delete"
     598    // there is output that needs flushing, so don't use runBasicProcess()
     599           
     600    } catch(Exception e) {
     601    log("@@@ Exception attempting to stop perl " + e.getMessage(), e);     
     602    }
     603}
     604
     605
     606// On linux and mac, p.destroy() suffices to kill processes launched by p as well.
     607// On Windows we need to do more work, since otherwise processes launched by p remain around executing until they naturally terminate.
     608// e.g. full-import.pl may be terminated with p.destroy(), but it launches import.pl which is left running until it naturally terminates.
     609static void destroyProcess(Process p) {
     610    // If it isn't windows, process.destroy() terminates any child processes too
     611    if(!Misc.isWindows()) {
     612    p.destroy();
     613    return;
     614    }   
     615   
     616    if(!SafeProcess.isAvailable("wmic")) {
     617    log("wmic, used to kill subprocesses, is not available. Unable to terminate subprocesses...");
     618    log("Kill them manually from the TaskManager or they will proceed to run to termination");
     619   
     620    // At least we can get rid of the top level process we launched
     621    p.destroy();
     622    return;
     623    }   
     624   
     625    // get the process id of the process we launched,
     626    // so we can use it to find the pids of any subprocesses it launched in order to terminate those too.
     627   
     628    long processID = SafeProcess.getProcessID(p);       
     629    log("Attempting to terminate sub processes of Windows process with pid " + processID);
     630    terminateSubProcessesRecursively(processID, p);
     631   
     632}
     633
     634// Helper function. Only for Windows.
     635// Counterintuitively, we're be killing all parent processess and then all child procs and all their descendants
     636// as soon as we discover any further process each (sub)process has launched. The parent processes are killed
     637// first in each case for 2 reasons:
     638// 1. on Windows, killing the parent process leaves the child running as an orphan anyway, so killing the
     639// parent is an independent action, the child process is not dependent on the parent;
     640// 2. Killing a parent process prevents it from launching further processes while we're killing off each child process
     641private static void terminateSubProcessesRecursively(long parent_pid, Process p) {
     642   
     643    // Use Windows wmic to find the pids of any sub processes launched by the process denoted by parent_pid
     644    SafeProcess proc = new SafeProcess("wmic process where (parentprocessid="+parent_pid+") get processid");
     645    proc.setSplitStdOutputNewLines(true); // since this is windows, splits lines by \r\n
     646    int exitValue = proc.runProcess(); // exitValue (%ERRORLEVEL%) is 0 either way.
     647    //log("@@@@ Return value from proc: " + exitValue);
     648   
     649    // need output from both stdout and stderr: stderr will say there are no pids, stdout will contain pids
     650    String stdOutput = proc.getStdOutput();
     651    String stdErrOutput = proc.getStdError();
     652   
     653   
     654    // Now we know the pids of the immediate subprocesses, we can get rid of the parent process
     655    // We know the children remain running: since the whole problem on Windows is that these
     656    // child processes remain running as orphans after the parent is forcibly terminated.
     657    if(p != null) { // we're the top level process, terminate the java way
     658    p.destroy();
     659    } else { // terminate windows way       
     660    SafeProcess.killWinProcessWithID(parent_pid); // get rid off current pid       
     661    }
     662   
     663    // parse the output to get the sub processes' pids 
     664    // Output looks like:
     665    // ProcessId
     666    // 6040
     667    // 180
     668    // 4948
     669    // 1084
     670    // 6384
     671    // If no children, then STDERR output starts with the following, possibly succeeded by empty lines:
     672    // No Instance(s) Available.
     673   
     674    // base step of the recursion
     675    if(stdErrOutput.indexOf("No Instance(s) Available.") != -1) {
     676    //log("@@@@ Got output on stderr: " + stdErrOutput);
     677    // No further child processes. And we already terminated the parent process, so we're done
     678    return;
     679    } else {
     680    //log("@@@@ Got output on stdout:\n" + stdOutput);
     681   
     682    // http://stackoverflow.com/questions/691184/scanner-vs-stringtokenizer-vs-string-split
     683   
     684    // find all childprocesses for that pid and terminate them too:
     685    Stack<Long> subprocs = new Stack<Long>();
     686    Scanner sc = new Scanner(stdOutput);
     687    while (sc.hasNext()) {
     688        if(!sc.hasNextLong()) {
     689        sc.next(); // discard the current token since it's not a Long
     690        } else {
     691        long child_pid = sc.nextLong();
     692        subprocs.push(new Long(child_pid));         
     693        }
     694    }
     695    sc.close();     
     696   
     697    // recursion step if subprocs is not empty (but if it is empty, then it's another base step)
     698    if(!subprocs.empty()) {
     699        long child_pid = subprocs.pop().longValue();
     700        terminateSubProcessesRecursively(child_pid, null);
     701    }   
     702    }
     703}
     704
     705// This method should only be called on a Windows OS
     706private static String getWinProcessKillCmd(Long processID) {
     707    // check if we first need to init WIN_KILL_CMD. We do this only once, but can't do it in a static codeblock
     708   
     709    if(WIN_KILL_CMD == null) {     
     710    if(SafeProcess.isAvailable("wmic")) {
     711        // https://isc.sans.edu/diary/Windows+Command-Line+Kung+Fu+with+WMIC/1229
     712        WIN_KILL_CMD = "wmic process _PROCID_ delete"; // like "kill -9" on Windows
     713    }
     714    else if(SafeProcess.isAvailable("taskkill")) { // check if we have taskkill or else use the longstanding tskill
     715       
     716        WIN_KILL_CMD = "taskkill /f /t /PID _PROCID_"; // need to forcefully /f terminate the process               
     717            //  /t "Terminates the specified process and any child processes which were started by it."
     718            // But despite the /T flag, the above doesn't kill subprocesses.
     719    }
     720    else { //if(SafeProcess.isAvailable("tskill")) { can't check availability since "which tskill" doesn't ever succeed
     721        WIN_KILL_CMD = "tskill _PROCID_"; // https://ss64.com/nt/tskill.html
     722    }       
     723    }
     724   
     725    if(WIN_KILL_CMD == null) { // can happen if none of the above cmds were available
     726    return null;
     727    }
     728    return WIN_KILL_CMD.replace( "_PROCID_", Long.toString(processID) );
     729}
     730
     731
     732// Run `which` on a program to find out if it is available. which.exe is included in winbin.
     733// On Windows, can use where or which. GLI's file/FileAssociationManager.java used which, so we stick to the same.
     734// where is not part of winbin. where is a system command on windows, but only since 2003, https://ss64.com/nt/where.html
     735// There is no `where` on Linux/Mac, must use which for them.
     736// On windows, "which tskill" fails but "which" succeeds on taskkill|wmic|browser names.
     737public static boolean isAvailable(String program) {     
     738    try {
     739    // On linux `which bla` does nothing, prompt is returned; on Windows, it prints "which: no bla in"
     740    // `which grep` returns a line of output with the path to grep. On windows too, the location of the program is printed
     741    SafeProcess prcs = new SafeProcess("which " + program);     
     742    prcs.runProcess();
     743    String output = prcs.getStdOutput();
     744    if(output.equals("")) {
     745        return false;
     746    }
     747    //System.err.println("*** 'which " + program + "' returned: " + output);
     748    return true;
     749    } catch (Exception exc) {
     750    return false;
     751    }
     752}   
     753   
     754// Google Java external process destroy kill subprocesses
     755// https://zeroturnaround.com/rebellabs/how-to-deal-with-subprocesses-in-java/ 
     756   
    429757//******************** Inner class and interface definitions ********************//
    430758// Static inner classes can be instantiated without having to instantiate an object of the outer class first
     
    515843    this.is = is;
    516844    this.split_newlines = split_newlines;
     845   
    517846    }
    518847   
     
    644973        // to already send EOF silently.
    645974       
    646         /*if(Utility.isWindows()) {
     975        /*if(Misc.isWindows()) {
    647976          osw.write("\032"); // octal for Ctrl-Z, EOF on Windows
    648977          } else { // EOF on Linux/Mac is Ctrl-D
     
    6781007    // logger and DebugStream print commands are synchronized, therefore thread safe.
    6791008    public static void log(String msg) {
     1009    if(DEBUG == 0) return;
    6801010    logger.info(msg);
    6811011
     
    6861016
    6871017    public static void log(String msg, Exception e) { // Print stack trace on the exception
     1018    if(DEBUG == 0) return;
    6881019    logger.error(msg, e);
    6891020
     
    6961027
    6971028    public static void log(Exception e) {
     1029    if(DEBUG == 0) return;     
    6981030    logger.error(e);
    6991031
  • main/trunk/greenstone3/web/WEB-INF/lib/cp.mf

    r22143 r31663  
    11Class-Path: LuceneWrapper.jar activation.jar axis-ant.jar
    22 axis.jar bsf.jar commons-discovery-0.2.jar
    3  commons-logging-1.0.4.jar derby.jar gsdl3.jar jaxp.jar
    4  jaxrpc.jar js.jar junit.jar log4j-1.2.8.jar lucene-1.4.1.jar
     3 commons-logging-1.0.4.jar derby.jar gsdl3.jar jaxp.jar jaxrpc.jar
     4 jna.jar jna-platform.jar js.jar junit.jar log4j-1.2.8.jar lucene-1.4.1.jar
    55 mail.jar saaj.jar wsdl4j-1.5.1.jar xercesImpl.jar
    66 xmlsec-1.2.1.jar gutil.jar
Note: See TracChangeset for help on using the changeset viewer.