Changeset 99 for trunk/gsdl/src

Timestamp:

1999-01-08T13:33:50+13:00 (25 years ago)

Author:

rjmcnab

Message:

Enabled mg and the library software to read in more than one index
at a time.

Location:

trunk/gsdl/src/library

Files:

• Makefile (modified) (1 diff)
• libinterface.cpp (modified) (4 diffs)
• libinterface.h (modified) (1 diff)
• mgq.c (modified) (10 diffs)
• mgq.h (modified) (1 diff)
• mgsearch.cpp (modified) (7 diffs)
• mgsearch.h (modified) (1 diff)

trunk/gsdl/src/library/Makefile

@@ -26 +26 @@
 CC = gcc
 CCFLAGS = -g
-DEFS = -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM -DUSE_FASTCGI
+DEFS = -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM -DUSE_FASTCGI -DGSDLSERVER
 RANLIB = ranlib
 INCLUDES = -I../../lib -I../../packages/mg-1.3d -I../../packages/mg-1.3d/lib \

trunk/gsdl/src/library/libinterface.cpp

@@ -268 +268 @@
   } else {
     default_index = real2dirindex (cfg_info.indexmap, cfg_info.defaultindex);
+  }
+
+  // set the text default index (the default index to use when
+  // retrieving documents).
+  text_default_index = default_index;
+  if (!isdoclevelindex (cfg_info.defaultindex)) {
+    text_default_index = real2dirindex (cfg_info.indexmap, 
+                    getdoclevelindex (cfg_info.indexmap));
   }
 
@@ -920 +928 @@
     // get document text if there is any
     if (info.contents.empty()) {
-        search.docTargetDocument(default_index, queryparams.collection,
+        search.docTargetDocument(text_default_index, queryparams.collection,
                  info.docnum, content);
     if (info.title != "<i>(introductory text)</i>")
@@ -966 +974 @@
     }
 
-    search.docTargetDocument(default_index, queryparams.collection,
+    search.docTargetDocument(text_default_index, queryparams.collection,
                  info.docnum, text);                    
     if (info.title != "<i>(introductory text)</i>") 
@@ -1015 +1023 @@
       // get document text if there is any
       if (info.contents.empty()) {
-        search.docTargetDocument(default_index, queryparams.collection,
+        search.docTargetDocument(text_default_index, queryparams.collection,
                                  info.docnum, content);
     if (info.title != "<i>(introductory text)</i>")

trunk/gsdl/src/library/libinterface.h

@@ -96 +96 @@
   int numbooks;
 
+  text_t text_default_index;
   text_t default_index;
 

trunk/gsdl/src/library/mgq.c

@@ -36 +36 @@
 #include "mgq.h"
 
+/* get a reasonable database cache size */
+#ifndef MAXNUMDATABASEINFO
+#  ifdef GSDLSERVER
+#    define MAXNUMDATABASEINFO 10
+#  else
+#    define MAXNUMDATABASEINFO 2
+#  endif
+#endif
+
+#define MAXCOLLECTIONLEN  16
+#define MAXMGDIRLEN       256
+#define MAXGENSUFFIXLEN   256
+#define MAXTEXTSUFFIXLEN  256
+
+typedef struct DatabaseInfo {
+  int accessnum; /* -1 = invalid record */
+  char collection[MAXCOLLECTIONLEN];
+  char mgdir[MAXMGDIRLEN];
+  char gensuffix[MAXGENSUFFIXLEN];
+  char textsuffix[MAXTEXTSUFFIXLEN];
+  query_data *qd;
+} DatabaseInfo;
+
+
 /* globals needed by some vague part of mg... */
 FILE *OutFile = NULL, *InFile = NULL;
@@ -41 +65 @@
 int Quitting = 0;
 
-static query_data *qd;
-char db_loaded_name[256] = "";
+/* globals needed to handle loading of databases */
+static int cur_cachenum = -1;
+
+/* globals needed by the database cache */
+static DatabaseInfo dbcache[MAXNUMDATABASEINFO];
+static int cache_nextaccessnum = 0;
+static int cache_numloaded = 0;
+
+
 
 #if defined(PARADOCNUM) ||  defined(NZDL)
-int *Paragraph = NULL;
-int Documents = 0;
-
-int GetDocNumFromParaNum(int paranum)
-{
+static int GetDocNumFromParaNum(query_data *qd, int paranum) {
+  int Documents = qd->td->cth.num_of_docs;
+  int *Paragraph = qd->paragraph;
   int low = 1, high = Documents;
   int mid = (low+high)/2;
@@ -117 +146 @@
       int docnum = GetDocNum(qd);
 #if defined(PARADOCNUM) ||  defined(NZDL)
-      if (qd->id->ifh.InvfLevel == 3) docnum = GetDocNumFromParaNum(docnum);
+      if (qd->id->ifh.InvfLevel == 3) docnum = GetDocNumFromParaNum(qd, docnum);
 #endif
       switch (kind)
@@ -147 +176 @@
 
 
-void
-send_query_term_freqs(QueryTermList *qtl,
-              int (*sender)(char *,int,int,float,void *), void *ptr)
+static void send_query_term_freqs(QueryTermList *qtl,
+                  int (*sender)(char *,int,int,float,void *), void *ptr)
 {
   int i;
@@ -162 +190 @@
 
 
-void
-send_terms (TermList * qtl, int (*sender)(char *,int,int,float,void *), void *ptr)
+static void send_terms (TermList * qtl, 
+            int (*sender)(char *,int,int,float,void *), void *ptr)
 {
   int i = 0;
@@ -206 +234 @@
 
 
-int mgq_init(char *mgdir, char *gensuffix, char *textsuffix)
-{
+
+
+
+
+/******************************************
+ * functions to handle the database cache *
+ ******************************************/
+
+/* init_dbcache should be called at the start of each */
+/* function which deals with the database cache */
+static void init_dbcache () {
+  static int dbcacheinited = 0;
+  int i;
+
+  if (dbcacheinited) return;
+
+  cache_numloaded = 0;
+
+  for (i=0; i<MAXNUMDATABASEINFO; i++) {
+    dbcache[i].accessnum = -1;
+    dbcache[i].collection[0] = '\0';
+    dbcache[i].mgdir[0] = '\0';
+    dbcache[i].gensuffix[0] = '\0';
+    dbcache[i].textsuffix[0] = '\0';
+    dbcache[i].qd = NULL;
+  }
+
+  dbcacheinited = 1;
+}
+
+/* returns the next cache access number and increments it */
+static int get_next_accessnum () {
+  return cache_nextaccessnum++;
+}
+
+/* get_free_dbcache returns the cache number which */
+/* was used the longest time ago */
+/* init_dbcache should be called before this function */
+static int get_free_dbcache () {
+  int i;
+  int minaccessnum = cache_nextaccessnum; /* the current max */
+  int minpos = 0;
+
+  for (i=0; i<MAXNUMDATABASEINFO; i++) {
+    if (dbcache[i].accessnum < minaccessnum) {
+      minaccessnum = dbcache[i].accessnum;
+      minpos = i;
+    }
+  }
+  
+  return minpos;
+}
+
+/* search_doc_collect will search for an index which */
+/* belongs to a certain collection and which has a document */
+/* level index. It returns -1 if none could be found. */
+/* init_dbcache should be called before this function */
+static int search_doc_collect (char *collection) {
+  int i;
+
+  for (i=0; i<MAXNUMDATABASEINFO; i++) {
+    if ((dbcache[i].accessnum >= 0) &&
+    (dbcache[i].qd != NULL) &&
+    (strcmp (collection, dbcache[i].collection) == 0) &&
+    (dbcache[i].qd->id->ifh.InvfLevel == 2)) {
+      dbcache[i].accessnum = get_next_accessnum ();
+      return i;
+    }
+  }
+
+  return -1;
+}
+
+/* search_doc_collect will search for an index which */
+/* has a certain gensuffix. It returns -1 if none could be found. */
+/* init_dbcache should be called before this function */
+static int search_gensuffix (char *gensuffix) {
+  int i;
+
+  for (i=0; i<MAXNUMDATABASEINFO; i++) {
+    if ((dbcache[i].accessnum >= 0) &&
+    (dbcache[i].qd != NULL) &&
+    (strcmp (gensuffix, dbcache[i].gensuffix) == 0)) {
+      dbcache[i].accessnum = get_next_accessnum ();
+      return i;
+    }
+  }
+
+  return -1;
+}
+
+/* unload_database will unload a certain entry within */
+/* the database cache, clearing it for furture use. */
+static void unload_database (int i) {
+  /* check to see if it contains anything */
+  if (dbcache[i].accessnum < 0 || dbcache[i].qd == NULL) 
+    return;
+
+  /* unload all the query information  */
+  FinishQuerySystem(dbcache[i].qd);
+  
+  /* reset all the db info */
+  dbcache[i].accessnum = -1;
+  dbcache[i].collection[0] = '\0';
+  dbcache[i].mgdir[0] = '\0';
+  dbcache[i].gensuffix[0] = '\0';
+  dbcache[i].textsuffix[0] = '\0';
+  dbcache[i].qd = NULL;
+
+  cache_numloaded--;
+  if (cache_numloaded < 0) cache_numloaded = 0;
+}
+
+/* cache_database will store the information about */
+/* a database in the database cache. */
+static void cache_database (int i, char *collection, char *mgdir, char *gensuffix,
+                char *textsuffix, query_data *qd) {
+  /* make sure this entry has been unloaded first */
+  if (dbcache[i].accessnum >= 0 && dbcache[i].qd != NULL) 
+    unload_database (i);
+
+  /* store the db info */
+  dbcache[i].accessnum = get_next_accessnum ();
+  strcpy (dbcache[i].collection, collection);
+  strcpy (dbcache[i].mgdir, mgdir);
+  strcpy (dbcache[i].gensuffix, gensuffix);
+  strcpy (dbcache[i].textsuffix, textsuffix);
+  dbcache[i].qd = qd;
+
+  cache_numloaded++;
+}
+
+static void make_current (int i) {
+  /* see if it is the current index */
+  if (i == cur_cachenum) return;
+ 
+  /* unload the old index */
+  if (cur_cachenum >= 0) UninitEnv ();
+  cur_cachenum = -1;
+
+  /* make sure the new one is ok */
+  if (i < 0 || dbcache[i].accessnum < 0 || dbcache[i].qd == NULL)
+    return;
+
+  /* load the new one */
+
   /* Initialise the environment with default values */
   InitEnv ();
   
-  SetEnv("mgdir",mgdir,NULL);
-  SetEnv("mgname",gensuffix,NULL);
-  SetEnv("textname",textsuffix,NULL);
+  SetEnv("mgdir",dbcache[i].mgdir,NULL);
+  SetEnv("mgname",dbcache[i].gensuffix,NULL);
+  SetEnv("textname",dbcache[i].textsuffix,NULL);
   
   PushEnv ();
   
-  qd = InitQuerySystem (GetDefEnv ("mgdir", ""),
-            GetDefEnv ("mgname", ""),
-            GetDefEnv ("textname", NULL),
-            NULL);
-  
-  if (qd == NULL) return 0;  else return 1;
-}
+  cur_cachenum = i;
+}
+
+
+
+/********************
+ * public functions *
+ ********************/
 
 int mgq_ask(char *line)
 {
+  query_data *qd = NULL;
   char QueryType;  char OutputType;  char *LinePtr;
+
+  if (cur_cachenum == -1) return 0;
+  qd = dbcache[cur_cachenum].qd;
+  if (qd == NULL) return 0;
   
   ResetFileStats (qd);
@@ -311 +489 @@
 int mgq_numdocs(void)
 {
+  query_data *qd = NULL;
+  if (cur_cachenum == -1) return 0;
+  qd = dbcache[cur_cachenum].qd;
+  if (qd == NULL) return 0;
+
   if (qd->DL) return qd->DL->num;
   else return 0;
@@ -317 +500 @@
 int mgq_numterms(void)
 {
+  query_data *qd = NULL;
+  if (cur_cachenum == -1) return 0;
+  qd = dbcache[cur_cachenum].qd;
+  if (qd == NULL) return 0;
+
   if (qd->QTL) return qd->QTL->num;
   else return 0;
@@ -323 +511 @@
 int mgq_results(enum result_kinds kind,int skip,int howmany, int (*sender)(char *, int, int, float, void *), void *ptr)
 {
+  query_data *qd = NULL;
+  if (cur_cachenum == -1) return 0;
+  qd = dbcache[cur_cachenum].qd;
+  if (qd == NULL) return 0;
+
   if (qd->DL) {
     qd->doc_pos = 0;
@@ -331 +524 @@
 
 
-void mgq_finish(void)
-{
-  FinishQuerySystem(qd);
-  UninitEnv ();
-}
-
-int load_database(char *mgdir, char *gensuffix, char *textsuffix)
-{
-  if ((db_loaded_name[0]) != '\0' && (strcmp(gensuffix,db_loaded_name) != 0)) {
-    mgq_finish();
-    db_loaded_name[0] = '\0';
-  }
-  if ((db_loaded_name[0] == '\0') && mgq_init(mgdir, gensuffix, textsuffix)) {
-    strcpy(db_loaded_name, gensuffix);
-  }
-  return (db_loaded_name[0] != '\0');
-}
+int is_dbcache_full () {
+  init_dbcache ();
+  if (cache_numloaded >= MAXNUMDATABASEINFO) return 1;
+  return 0;
+}
+
+int load_database (char *collection, char *mgdir, 
+           char *gensuffix, char *textsuffix) {
+  int i;
+  query_data *qd = NULL;
+  FILE *deb = NULL;
+  init_dbcache ();
+
+  /* print out some debug information */
+/*  deb = fopen ("/home/rjmcnab/gsdl/etc/deb.txt", "a");
+  fprintf (deb, "\ncache_nextaccessnum: %i\n", cache_nextaccessnum);
+  fprintf (deb, "cache_numloaded: %i\n", cache_numloaded);
+  fprintf (deb, "cur_cachenum: %i\n", cur_cachenum);
+  fprintf (deb, "MAXNUMDATABASEINFO: %i\n\n", MAXNUMDATABASEINFO);
+  for (i=0; i<MAXNUMDATABASEINFO; i++) {
+    fprintf (deb, "Entry %i\n", i);
+    fprintf (deb, "  accessnum: %i\n", dbcache[i].accessnum);
+    fprintf (deb, "  collection: %s\n", dbcache[i].collection);
+    fprintf (deb, "  mgdir: %s\n", dbcache[i].mgdir);
+    fprintf (deb, "  gensuffix: %s\n", dbcache[i].gensuffix);
+    fprintf (deb, "  textsuffix: %s\n", dbcache[i].textsuffix);
+    fprintf (deb, "  qd: %x\n", (int)(dbcache[i].qd));
+  }
+  fclose (deb); */
+
+  /* search for the index */
+  i = search_gensuffix (gensuffix);
+  if (i > 0) {
+    make_current (i);
+    return 1;
+  }
+
+  /* if there was a current database then the */
+  /* environment needs uninitialising */
+  make_current (-1);
+
+  /* get a free cache number */
+  i = get_free_dbcache ();
+  unload_database (i);
+
+  /* load the index */
+  qd = InitQuerySystem (mgdir, gensuffix, textsuffix, NULL);
+  if (qd == NULL) return 0;
+
+  /* cache this index */
+  cache_database (i, collection, mgdir, gensuffix, textsuffix, qd);
+
+  /* make this index current */
+  make_current (i);
+
+  return 1;
+}
+
+/* load_text_database tries to make a level 2 index of the */
+/* specified collection current */
+int load_text_database (char *collection) {
+  int i;
+  init_dbcache ();
+
+  /* search for the index */
+  i = search_doc_collect (collection);
+
+  /* return if none were found */
+  if (i < 0) return 0;
+  
+  /* make this index current */
+  make_current (i);
+  return 1;
+}
+
+void close_all_databases () {
+  int i;
+  init_dbcache ();
+  
+  /* unload all active databases */
+  for (i=0; i<MAXNUMDATABASEINFO; i++) {
+    unload_database (i);
+  }
+
+  /* if there was a current database then the */
+  /* environment needs uninitialising */
+  make_current (-1);
+}

trunk/gsdl/src/library/mgq.h

@@ -14 +14 @@
 int mgq_numdocs(void);
 int mgq_numterms(void);
-int load_database(char *mgdir, char *gensuffix, char *textsuffix);
 
-/* this will equal "" when a database is not loaded */
-/* if a database is loaded it will equal its gensuffix */
-extern char db_loaded_name[256];
+
+int is_dbcache_full ();
+int load_database (char *collection, char *mgdir, char *gensuffix, char *textsuffix);
+int load_text_database (char *collection);
+void close_all_databases ();
 
 

trunk/gsdl/src/library/mgsearch.cpp

@@ -139 +139 @@
 }
 
+bool isdoclevelindex (const text_t &realindex) {
+  char *docstr = "document";
+  text_t::const_iterator here = realindex.begin ();
+  text_t::const_iterator end = realindex.end ();
+
+  while (here != end) {
+    if (*docstr == '\0') return true;
+    if (*docstr != (char)(*here)) return false;
+    docstr++;
+    here++;
+  }
+
+  return false;
+}
+
+text_t getdoclevelindex (const text_tarray &indexmap) {
+  text_tarray::const_iterator here = indexmap.begin();
+  text_tarray::const_iterator end = indexmap.end();
+  text_t maprealindex, mapdirindex;
+  
+  while (here != end) {
+    getrealdirindex (*here, maprealindex, mapdirindex);
+    if (isdoclevelindex (maprealindex)) return maprealindex;
+    here++;
+  }
+  
+  return "";
+}
 
 
@@ -277 +305 @@
 
 
-  // get the names of the index and text suffixes
+  // get the names of the collection, index and text suffixes
+  char *ccollection = queryparams.collection.getcstr();
+  assert (ccollection != NULL);
   char *idxsuffix = (getindexsuffix (queryparams.collection, 
                      queryparams.search_index)).getcstr();
@@ -290 +320 @@
 #endif
 
-  if (load_database(ccollectdir, idxsuffix, txtsuffix))
+  if (load_database(ccollection, ccollectdir, idxsuffix, txtsuffix))
     {
       setsearchmode (queryparams);
@@ -299 +329 @@
 
   // free up the c strings
+  delete ccollection;
   delete idxsuffix;
   delete txtsuffix;
@@ -441 +472 @@
                       text_t &output)
 {
-  bool databaseloaded = true;
+  int databaseloaded = 0;
 
   output.clear();
 
-
-  // make sure index is level 2
-
-  ////// this changed with new naming scheme in new building software
-  /////  i.e paragraph level index no longer contain number '3' but begin
-  /////  with letter 'p'
-
-  text_t db_loaded = db_loaded_name;
-
-  if (!db_loaded.empty()) {
-      text_t::const_iterator here = db_loaded.begin();
-      text_t::const_iterator end = db_loaded.end();
-  
-      
-      //while (here != end) {
-      //  if (*here == '3')
-      //    databaseloaded = false;
-      //  here ++;
-      //}
-
-      char separator = '/';
-      text_t db;
-      int found = 0;
-#ifdef __WIN32__
-      separator = '\\';
-#endif;
-      // strip away path to db and following collection name
-      end --;
-      while (end != here) {
-          if (*end == separator) {
-              if (found) break;
-              else {db.clear(); found = 1; end--; continue;}
-          }
-          db.push_back(*end);
-          end --;
-      }
-
-      // string will have been reversed above so see if last 
-      // character is 'p'
-      if (db[db.size()-1] == 'p') databaseloaded = false;
-  }
-
-  // find out if the database is already loaded
-  // this is needed because a different index (but valid one)
-  // might be already loaded.
-  // this comparison is needed because 'load_database'
-  // is now more oriented towards indexes
-  if (databaseloaded == true) {
-      text_t::const_iterator here = collection.begin();
-      text_t::const_iterator end = collection.end();
-      char *dbhere = &db_loaded_name[strlen(db_loaded_name) - collection.size()]; // assumes collection shorter than db_loaded_name
-      while (here != end)
-      {
-          if (*here != *dbhere)
-          {
-              databaseloaded = false;
-              break;
-          }
-          here++;
-          dbhere++;
-      }
-  }
+  char *ccollection = collection.getcstr();
+  assert (ccollection != NULL);
+
+  // see if we can make an appropriate database current
+  databaseloaded = load_text_database (ccollection);
 
   // try and load the database
@@ -524 +498 @@
 #endif
       
-      if (load_database(ccollectdir, idxsuffix, txtsuffix))
-    databaseloaded = true;
-      else 
-    databaseloaded = false;
+      databaseloaded = load_database(ccollection, ccollectdir, idxsuffix, txtsuffix);
       
       // free up the c strings
@@ -534 +505 @@
       delete ccollectdir;
     }
+
+  // free up the c collection string
+  delete ccollection;
   
   if (databaseloaded)

trunk/gsdl/src/library/mgsearch.h

@@ -19 +19 @@
 text_t real2dirindex (const text_tarray &indexmap, const text_t &realindex);
 text_t real2macroindex (const text_t &realindex);
+bool isdoclevelindex (const text_t &realindex);
+text_t getdoclevelindex (const text_tarray &indexmap);
 
 class mgsearchclass {