source: trunk/gsdl3/src/packages/mg/jni/MGWrapperImpl.c@ 3743

/*
 *    MGWrapperImpl.c
 *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
 *
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */


#include "MGWrapperImpl.h"
#include <jni.h>
#include "org_greenstone_mg_MGWrapper.h"

#include "backend.h"
#include "environment.h"


/*************************************************************************
   NOTES

 - Features supported by MGPP but not by MG (AFAIK)

   - Sorting results by rank.
     Done here as a post-processing operation. Could be more efficient in
     some cases: the current solution is not very good when the number of
     matching documents is large and the number of desired matches is
     small. In this case it would be better to iterate through the array
     picking out the best documents rather than sorting them all.

   - Asking for query term frequencies to be returned.
     This cannot be turned off in MG. If the term frequencies are not
     required, they are simply not passed bassed to the Java side.

   - Choosing the index for queries.
     It is possible for MG to build Section and Document indexes (for
     example), but these are two separate indexes, and must be loaded
     separately. This module can load more than one index at a time, thus
     processing queries to different indexes more quickly.
     NOTE: This replaces TWO options in the MGPP version: returnLevel and
           queryLevel.

   - "Some" (boolean OR) queries.
     In boolean querying mode, MG assumes query terms are joined by ANDs.
     There is no way to tell MG to treat the query terms as if they were
     joined by ORs. Here, "some" queries are handled by explicitly
     inserting OR ("|") characters between the query terms.

 *************************************************************************
   TO DO

   - Find out about necessity of other query term information
     (eg. equivalent terms).

 *************************************************************************/


#define MAX_INDEXES_CACHED 2


/*********************************************
   initialisation stuff
 *********************************************/

/* cached ids for java stuff */
jfieldID FID_mg_data = NULL; /* MGWrapperData */
jfieldID FID_query_result = NULL; /* MGQueryResult */
jmethodID MID_addDoc = NULL; /* MGQueryResult.addDoc() */
jmethodID MID_addTerm = NULL; /* MGQueryResult.addTerm() */
jmethodID MID_setTotalDocs = NULL; /* MGQueryResult.setTotalDocs() */
jmethodID MID_clearResult = NULL; /* MGQueryResult.clear() */


/* to access objects and methods on java side, need their field/method ids - 
 this initialises them at the start to avoid recalculating them each time they
 are needed
Note: the descriptors need to be exactly right, otherwise you get an error 
saying "no such field" but no reference to the fact that it has the right 
name but the wrong type.
Note: apparently the jclass is a local ref and should only work 
in the method that created it. It seems to work ok, but I'll make it
global cos the book said I should, and it may avoid future hassles.
*/
JNIEXPORT void JNICALL
Java_org_greenstone_mg_MGWrapper_initIDs(JNIEnv *j_env, jclass j_cls)
{
  jclass JC_MGQueryResult;

  /* a long-"J" */
  FID_mg_data = (*j_env)->GetFieldID(j_env, j_cls, "mg_data_ptr_", "J");
  assert(FID_mg_data != NULL);

  /* an object -"L<class name>;" */
  FID_query_result = (*j_env)->GetFieldID(j_env, j_cls, "mg_query_result_",
                      "Lorg/greenstone/mg/MGQueryResult;");
  assert(FID_query_result != NULL);

  /* the methods we want to use */
  JC_MGQueryResult = (*j_env)->FindClass(j_env, "org/greenstone/mg/MGQueryResult");

  /* addDoc(long doc, float rank) */
  MID_addDoc = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "addDoc", "(JF)V");
  assert(MID_addDoc != NULL);

  /* addTerm(String term, String tag, int stem, long match, long freq, String[] equivs) */
  MID_addTerm = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "addTerm", "(Ljava/lang/String;Ljava/lang/String;IJJ[Ljava/lang/String;)V");
  assert(MID_addTerm != NULL);

  /* setTotalDocs(long) */
  MID_setTotalDocs = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "setTotalDocs", "(J)V");
  assert(MID_setTotalDocs != NULL);

  /* clear(void) */
  MID_clearResult = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "clear", "()V");
  assert(MID_clearResult != NULL);
}


/* the java side MGWrapper has a pointer to a C object - MGWrapperData
   initialise this and set the pointer 
*/
JNIEXPORT jboolean JNICALL
Java_org_greenstone_mg_MGWrapper_initCSide(JNIEnv *j_env, jobject j_obj)
{
  /* Allocate a MGWrapperData object to store query parameters */
  MGWrapperData* data = (MGWrapperData*) malloc(sizeof(MGWrapperData));
  assert(data != NULL);

  /* Set default values - no stemming, no case-folding, boolean OR queries */
  data->defaultStemMethod = 0;
  data->defaultBoolCombine = 0;

  /* Allocate a QueryInfo object to store more query parameters */
  data->queryInfo = (QueryInfo*) malloc(sizeof(QueryInfo));
  assert(data->queryInfo != NULL);

  /* Set default values - 50 documents max, return term freqs, sort by rank */
  data->queryInfo->index = NULL;
  data->queryInfo->maxDocs = 50;
  data->queryInfo->needTermFreqs = 1;
  data->queryInfo->sortByRank = 1;

  /* Save the object on the Java side */
  (*j_env)->SetIntField(j_env, j_obj, FID_mg_data, (long) data);

  /* Initialise MG environment variables */
  InitEnv();
  SetEnv("expert", "true", NULL);
  SetEnv("mode", "docnums", NULL);
  return 1;  /* true - no errors */
}


/*******************************************
   Index caching
 *******************************************/

query_data* cached_indexes[MAX_INDEXES_CACHED] = { NULL };


/* Get the index data necessary to perform a query or document retrieval */
query_data*
loadIndexData(char* base_dir, char* index_path, char* text_path)
{
  char* index_path_name;
  char* text_path_name;
  query_data* qd;
  int i = 0;

  /* Form the path name of the desired indexes */
  index_path_name = (char*) malloc(strlen(base_dir) + strlen(index_path) + 1);
  assert(index_path_name != NULL);
  strcpy(index_path_name, base_dir);
  strcat(index_path_name, index_path);
  printf("Index pathname: %s\n", index_path_name);

  text_path_name = (char*) malloc(strlen(base_dir) + strlen(text_path) + 1);
  assert(text_path_name != NULL);
  strcpy(text_path_name, base_dir);
  strcat(text_path_name, text_path);
  printf("Text pathname: %s\n", text_path_name);

  /* Search through the cached indexes for the desired one */
  while (i < MAX_INDEXES_CACHED && cached_indexes[i] != NULL) {
    printf("(Cached) Pathname: %s\n", cached_indexes[i]->pathname);
    printf("(Cached) Textpathname: %s\n", cached_indexes[i]->textpathname);

    /* Check if the index has already been loaded */
    if ((strcmp(index_path_name, cached_indexes[i]->pathname) == 0) &&
    (strcmp(text_path_name, cached_indexes[i]->textpathname) == 0)) {
      /* Index has already been loaded and cached, so return it */
      printf("Found index!\n");
      free(index_path_name);
      free(text_path_name);
      return cached_indexes[i];
    }

    i++;
  }

  /* Text strings no longer needed */
  free(index_path_name);
  free(text_path_name);

  /* The index is not cached, so load it now */
  qd = InitQuerySystem(base_dir, index_path, text_path, NULL);
  if (!qd) {
    printf("Error: Could not InitQuerySystem()...\n");
    return NULL;
  }

  /* The index loaded OK, so cache it */
  /* This could be a little more sophisticated, eg. replace least frequently used index */
  if (i >= MAX_INDEXES_CACHED)
    i = MAX_INDEXES_CACHED - 1;

  /* Free the index being replaced */
  if (cached_indexes[i] != NULL)
    FinishQuerySystem(cached_indexes[i]);

  /* Cache the loaded index, and return it */
  cached_indexes[i] = qd;
  return cached_indexes[i];
}


/* Clean up by unloading all cached indexes */
JNIEXPORT jboolean JNICALL
Java_org_greenstone_mg_MGWrapper_unloadIndexData(JNIEnv* j_env, jobject j_obj)
{
  /* Free all the loaded indexes */
  int i = 0;
  while (i < MAX_INDEXES_CACHED && cached_indexes[i] != NULL) {
    FinishQuerySystem(cached_indexes[i]);
    cached_indexes[i] = NULL;
    i++;
  }

  return 1;  /* true - no errors */
}


/****************************************************
   retrieve a document
 ****************************************************/

/* Returns a document from mg as a string */
JNIEXPORT jstring JNICALL
Java_org_greenstone_mg_MGWrapper_getDocument(JNIEnv *j_env, jobject j_obj,
                         jstring j_base_dir, jstring j_text_path,
                         jlong j_docnum)
{
  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);

  char* index_path;
  const char* base_dir;
  const char* text_path;
  query_data* qd;

  u_long pos, len;
  u_char* c_buffer = NULL;
  u_char* uc_buffer = NULL;
  int ULen;

  jstring result;

  /* Make sure an index has been specified */
  index_path = data->queryInfo->index;
  assert(index_path != NULL);

  /* Obtain C versions of the two string parameters */
  base_dir = (*j_env)->GetStringUTFChars(j_env, j_base_dir, NULL);
  if (base_dir == NULL) {
    return NULL;
  }
  text_path = (*j_env)->GetStringUTFChars(j_env, j_text_path, NULL);
  if (text_path == NULL) {
    (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
    return NULL;
  }

  /* Load the appropriate index for satisfying this request */
  qd = loadIndexData((char*) base_dir, (char*) index_path, (char*) text_path);

  /* The C text strings are no longer needed */
  (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
  (*j_env)->ReleaseStringUTFChars(j_env, j_text_path, text_path);

  /* Check that the index was loaded successfully */
  assert(qd != NULL);

  /* Get the document position and length in the text file */
  printf("Fetching document number %ld...\n", (unsigned long) j_docnum);
  FetchDocStart(qd, (unsigned long) j_docnum, &pos, &len);
  printf("Fetched document start. Pos: %ld, Len: %ld\n", pos, len);

  /* Allocate memory for the document text (from mg/src/text/mgquery.c:RawDocOutput()) */
  c_buffer = (u_char*) malloc(len);
  assert(c_buffer != NULL);
  uc_buffer = (u_char*) malloc((int) (qd->td->cth.ratio * 1.01 * len) + 100);
  assert(uc_buffer != NULL);

  /* Seek to the correct position in the file and read the document text */
  Fseek (qd->td->TextFile, pos, 0);
  Fread (c_buffer, 1, len, qd->td->TextFile);

  /* Decode (?) the document text into another buffer, and terminate it */
  DecodeText (qd->cd, c_buffer, len, uc_buffer, &ULen);
  uc_buffer[ULen] = '\0';

  /* Load the document text into a Java string */
  result = (*j_env)->NewStringUTF(j_env, uc_buffer);
  assert(result != NULL);

  /* Free C buffers */
  free(c_buffer);
  free(uc_buffer);

  /* Return the document text */
  return result;
}


/*******************************************
   do a query
 *******************************************/

/* Comparison function for sorting documents by their weight (decreasing order) */
int
document_weight_comp(const void *A, const void *B)
{
  const DocEntry *a = A;
  const DocEntry *b = B;

  /* Compare on weight, highest wins */
  if (a->Weight < b->Weight)
    return  1;
  if (a->Weight > b->Weight)
    return -1;
  return 0;
}


/* do the actual query - the results are written to query_result held on the Java side */
JNIEXPORT void JNICALL
Java_org_greenstone_mg_MGWrapper_runQuery(JNIEnv *j_env, jobject j_obj,
                      jstring j_base_dir, jstring j_text_path,
                      jstring j_query)
{
  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);

  char* index_path;
  const char* base_dir;
  const char* text_path;
  query_data* qd;

  jobject result_ptr;
  char* query;
  BooleanQueryInfo bqi;
  int i, j;

  jthrowable exc;

  /* Make sure an index has been specified */
  index_path = data->queryInfo->index;
  assert(index_path != NULL);

  /* Obtain C versions of the two string parameters */
  base_dir = (*j_env)->GetStringUTFChars(j_env, j_base_dir, NULL);
  if (base_dir == NULL) {
    return;
  }
  text_path = (*j_env)->GetStringUTFChars(j_env, j_text_path, NULL);
  if (text_path == NULL) {
    (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
    return;
  }

  /* Load the appropriate index for satisfying this request */
  qd = loadIndexData((char*) base_dir, (char*) index_path, (char*) text_path);

  /* The C text strings are no longer needed */
  (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
  (*j_env)->ReleaseStringUTFChars(j_env, j_text_path, text_path);

  /* Check that the index was loaded successfully */
  assert(qd != NULL);

  /* Remove anything hanging around from last time */
  FreeQueryDocs(qd);

  /* The result to write to */
  result_ptr = (*j_env)->GetObjectField(j_env, j_obj, FID_query_result);
  assert(result_ptr != NULL);

  /* Clear any previous query results */
  (*j_env)->CallVoidMethod(j_env, result_ptr, MID_clearResult);
  exc = (*j_env)->ExceptionOccurred(j_env);
  if (exc) {
    (*j_env)->ExceptionDescribe(j_env);
    return;
  }

  /* Obtain a C version of the query string */
  query = (char*) (*j_env)->GetStringUTFChars(j_env, j_query, NULL);
  assert(query != NULL);
  printf("Searching for query \"%s\"...\n", query);

  /* Make sure the query isn't empty */
  if (strlen(query) == 0) {
    printf("Warning: Empty query.\n");
    return;
  }

  /* Boolean OR ("some") queries: must manually insert OR ("|") tokens */
  if (data->defaultBoolCombine == 0) {
    int in_space = 0;
    for (i = 0; i < strlen(query); i++) {
      if (in_space) {
    if (query[i] == '|') /* OR character already inserted, so remove others */
      query[i] = ' ';
    else if (!isspace(query[i])) /* moving out of a space region */
      in_space = 0;
      }
      else if (!in_space && isspace(query[i])) { /* moving into a space region */
    in_space = 1;
    query[i] = '|'; /* insert an OR character */
      }
    }
    printf("Boolean OR query string: %s\n", query);
  }

  /* If the documents need to be sorted by rank, all of them must be retrieved */
  bqi.MaxDocsToRetrieve = ((data->queryInfo->sortByRank) ? -1 : data->queryInfo->maxDocs);

  /* Perform query */
  /* Had to add "words$o" to LIB_OBJS in mg/src/text/Makefile and recompile mg for this */
  BooleanQuery(qd, (char*) query, &bqi, data->defaultStemMethod);

  /* Finished with the C query string */
  (*j_env)->ReleaseStringUTFChars(j_env, j_query, query);

  /* Check the query was processed successfully */
  assert(qd->DL != NULL);
  assert(qd->QTL != NULL);
  assert(qd->TL != NULL);

  /* Record the total number of matching documents */
  (*j_env)->CallVoidMethod(j_env, result_ptr, MID_setTotalDocs, (jlong) qd->DL->num);
  exc = (*j_env)->ExceptionOccurred(j_env);
  if (exc) {
    (*j_env)->ExceptionDescribe(j_env);
    return;
  }

  /* Sort the documents by rank, if desired */
  if (data->queryInfo->sortByRank) {
    qsort(qd->DL->DE, qd->DL->num, sizeof(DocEntry), document_weight_comp);
  }

  /* Record the matching documents, but only the number requested */
  printf("Number of matching documents: %d\n", qd->DL->num);
  for (i = 0; (i < qd->DL->num && i < data->queryInfo->maxDocs); i++) {
    int doc_num = qd->DL->DE[i].DocNum;
    float doc_weight = qd->DL->DE[i].Weight;

    /* Call the addDoc function (Java side) to record a matching document */
    (*j_env)->CallVoidMethod(j_env, result_ptr, MID_addDoc,
                 (jlong) doc_num, (jfloat) doc_weight);
    exc = (*j_env)->ExceptionOccurred(j_env);
    if (exc) {
      (*j_env)->ExceptionDescribe(j_env);
      return;
    }
  }

  /* Record the term information, if desired */
  if (data->queryInfo->needTermFreqs) {
    printf("Number of terms: %d\n", qd->TL->num);
    printf("Number of query terms: %d\n", qd->QTL->num);

    /* Find each query term in the term list, and grab its frequency */
    for (i = 0; i < qd->QTL->num; i++) {
      unsigned int query_term_length = (unsigned int) qd->QTL->QTE[i].Term[0];
      unsigned char* query_term;
      jstring j_query_term;
      jint stem = qd->QTL->QTE[i].stem_method;
      jlong match;
      jlong freq;

      /* Allocate memory to store this query term, and fill it */
      query_term = (unsigned char*) malloc(query_term_length + 1);
      assert(query_term != NULL);
      strncpy(query_term, &(qd->QTL->QTE[i].Term[1]), query_term_length);
      query_term[query_term_length] = '\0';
      printf("Query term: %s\n", query_term);

      /* Allocate a new jstring for the query term */
      j_query_term = (*j_env)->NewStringUTF(j_env, query_term);
      assert(j_query_term != NULL);

      /* Find the matching term in the term list */
      for (j = 0; j < qd->TL->num; j++) {
    unsigned int term_length = (unsigned int) qd->TL->TE[j].Word[0];

    /* Stemming and case-folding mean both comparisons are necessary */
    if ((strncmp(query_term, &(qd->TL->TE[j].Word[1]), term_length) == 0) &&
        (strncmp(query_term, &(qd->TL->TE[j].Word[1]), query_term_length) == 0)) {
      /* Get the document count and total frequency of the term */
      match = qd->TL->TE[j].WE.doc_count;
      freq = qd->TL->TE[j].WE.count;

      /* Call the addTerm function (Java side) to record term information */
      (*j_env)->CallVoidMethod(j_env, result_ptr, MID_addTerm,
                   j_query_term, NULL, stem, match, freq, NULL);
      exc = (*j_env)->ExceptionOccurred(j_env);
      if (exc) {
        (*j_env)->ExceptionDescribe(j_env);
        return;
      }

      /* There can only be one match */
      break;
    }
      }

      /* Finished with this query term */
      free(query_term);
    }
  }
}


/*******************************************
   set query options
 *******************************************/

/* Turn casefolding on or off */
JNIEXPORT void JNICALL
Java_org_greenstone_mg_MGWrapper_setCase(JNIEnv *j_env, jobject j_obj, 
                     jboolean j_on)
{
  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);

  if (j_on) {
    data->defaultStemMethod |= 1;
  } else {
    data->defaultStemMethod &= 0xe;
  }
}


/* Turn stemming on or off */
JNIEXPORT void JNICALL
Java_org_greenstone_mg_MGWrapper_setStem(JNIEnv *j_env, jobject j_obj, 
                     jboolean j_on)
{
  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);

  if (j_on) {
    data->defaultStemMethod |= 2;
  } else {
    data->defaultStemMethod &= 0xd;
  }
}


/* Set the maximum number of documents to return from a query */
JNIEXPORT void JNICALL
Java_org_greenstone_mg_MGWrapper_setMaxDocs(JNIEnv *j_env, jobject j_obj, 
                        jint j_max)
{
  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
  data->queryInfo->maxDocs = j_max;
}


/* Turn sorting by rank on or off */
JNIEXPORT void JNICALL
Java_org_greenstone_mg_MGWrapper_setSortByRank(JNIEnv *j_env, jobject j_obj, 
                                               jboolean j_on)
{
  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);  
  data->queryInfo->sortByRank = j_on;
}


/* Turn term frequency recording on or off */
JNIEXPORT void JNICALL
Java_org_greenstone_mg_MGWrapper_setReturnTerms(JNIEnv *j_env, jobject j_obj, 
                        jboolean j_on)
{
  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
  data->queryInfo->needTermFreqs = j_on;
}


/* Choose MG index to search */
JNIEXPORT void JNICALL
Java_org_greenstone_mg_MGWrapper_setIndex(JNIEnv *j_env, jobject j_obj,
                      jstring j_index)
{
  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);

  /* Get the index name as a C string */
  const char* index = (*j_env)->GetStringUTFChars(j_env, j_index, NULL);
  assert(index != NULL);
  printf("Choosing index %s...\n", index);

  /* Free the previous index name */
  if (data->queryInfo->index)
    free(data->queryInfo->index);

  /* Allocate memory for the index name, and fill it */
  data->queryInfo->index = (char*) malloc(strlen(index) + 1);
  assert(data->queryInfo->index != NULL);
  strcpy(data->queryInfo->index, index);

  /* Release the index string */
  (*j_env)->ReleaseStringUTFChars(j_env, j_index, index);
}


/* Choose boolean AND or boolean OR queries */
JNIEXPORT void JNICALL 
Java_org_greenstone_mg_MGWrapper_setMatchMode(JNIEnv *j_env, jobject j_obj,
                          jint j_mode)
{
  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
  data->defaultBoolCombine = j_mode;
}


/* Get a text representation of the current parameter values */
JNIEXPORT jstring JNICALL
Java_org_greenstone_mg_MGWrapper_getQueryParams(JNIEnv *j_env, jobject j_obj)
{
  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
  char result[512];  /* Assume this is big enough */

  /* Print the data to a character array */
  sprintf(result, "Query params:\nindex\t\t%s\ncasefold\t%d\nstem\t\t%d\norder by rank\t%d\nquery type\t%s\nmax docs\t%d\n",
      (data->queryInfo->index == NULL ? "<none loaded>" : data->queryInfo->index),
      (data->defaultStemMethod & 1),
      (data->defaultStemMethod & 2),
      (data->queryInfo->sortByRank),
      (data->defaultBoolCombine == 1 ? "all" : "some"),
      (data->queryInfo->maxDocs));

  /* Convert to a jstring, and return */
  return (*j_env)->NewStringUTF(j_env, result);
}