source: trunk/gsdl/src/colservr/gdbmclass.cpp@ 543

/**********************************************************************
 *
 * gdbmclass.cpp -- 
 * Copyright (C) 1999  The New Zealand Digital Library Project
 *
 * A component of the Greenstone digital library software
 * from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 * $Id: gdbmclass.cpp 534 1999-09-07 04:57:43Z sjboddie $
 *
 *********************************************************************/

/*
   $Log$
   Revision 1.6  1999/09/07 04:57:21  sjboddie
   added gpl notice

   Revision 1.5  1999/01/25 03:59:40  sjboddie
   fixed a bug in sorting code

   Revision 1.4  1999/01/21 21:20:08  sjboddie
   removed unused collection parameter from several functions

   Revision 1.3  1999/01/19 01:38:15  rjmcnab

   Made the source more portable.

   Revision 1.2  1999/01/12 01:51:01  rjmcnab

   Standard header.

   Revision 1.1  1999/01/08 09:02:15  rjmcnab

   Moved from src/library.

 */


#include "text_t.h"
#include "gdbmclass.h"
#include "unitool.h"
#include "gsdlunicode.h"
#include "fileutil.h"
#include <ctype.h>
#include <string.h>

#if defined(GSDL_NEED_STRINGS_H)
#include <strings.h>
#endif

#if defined(GSDL_USE_OBJECTSPACE)
#  include <ospace\std\algorithm>
#elif defined(GSDL_USE_STL_H)
#  if defined(GSDL_USE_ALGO_H)
#    include <algo.h>
#  else
#    include <algorithm.h>
#  endif
#else
#  include <algorithm>
#endif


static int compare_str (const char *e1, const char *e2) {
#ifdef __WIN32__
  return _stricmp(e1, e2);
#else 
  return strcasecmp(e1, e2);  
#endif
}

static int compare_str (const void *e1, const void *e2) {
#ifdef __WIN32__
  return _stricmp(*((char**)e1), *((char**)e2));
#else 
  return strcasecmp(*((char**)e1), *((char**)e2));  
#endif
}



void gdbm_info::clear () {
  docnum = 0;             // 'd'
  title.clear();          // 't'
  parent.clear();         // 'p'
  classification.clear(); // 'x'
  contents.clear();       // 'c'
  jobnum.clear();         // 'j'
  OID.clear();            // 'o'
  author.clear();         // 'a'
  source.clear();         // 's'
  date.clear();           // 'i'
}





// returns 0 if failed, 1 if opened
int gdbmclass::opendatabase (const text_t &filename) {
  text_t data_location;
  int block_size = 0;
  
  if (gdbmfile != NULL) {
    if (openfile == filename) return 1;
    else closedatabase ();
  }

  openfile = filename;

  char *namebuffer = filename.getcstr();
  gdbmfile = gdbm_open (namebuffer, block_size, GDBM_READER, 00664, NULL);
  delete namebuffer;

  if (gdbmfile == NULL && logout != NULL) {
    outconvertclass text_t2ascii;
    (*logout) << text_t2ascii << "database open failed on: " << filename << "\n";
  }

  return (gdbmfile != NULL);
}


void gdbmclass::closedatabase () {
  if (gdbmfile == NULL) return;
  
  gdbm_close (gdbmfile);
  gdbmfile = NULL;
  openfile.clear();
}


// returns 0 on success, -1 on failure
// key and collection aren't references as they might be aliases to
// something in info
int gdbmclass::getinfo (text_t key, gdbm_info &info) {
  text_t data;

  if (!getkeydata (key, data)) return -1;
  text_t::iterator here = data.begin ();
  text_t::iterator end = data.end ();

  text_t ikey, ivalue;
  info.clear (); // reset info

  while (getinfoline(here, end, ikey, ivalue)) {
    if (ikey == "d") { info.docnum = ivalue.getint(); }
    else if (ikey == "t") { info.title = ivalue; }
    else if (ikey == "p") { info.parent = ivalue; }
    else if (ikey == "x") { info.classification = ivalue; }
    else if (ikey == "c") { info.contents = ivalue; } 
    else if (ikey == "j") { info.jobnum = ivalue; }
    else if (ikey == "o") { info.OID = ivalue; } 
    else if (ikey == "a") { info.author = ivalue; } 
    else if (ikey == "s") { info.source = ivalue; } 
    else if (ikey == "i") { info.date = ivalue; }
  }
  
  return 0;
}


// returns 1 if the key exists
int gdbmclass::exists (text_t key) {
  text_t data;
  return getkeydata (key, data);
}


// returns 1 if successful
int gdbmclass::getkeydata (text_t key, text_t &data) {
  datum key_data;
  datum return_data;

  if (gdbmfile == NULL) return 0;

  // get a utf-8 encoded c string of the unicode key
  key_data.dptr = (to_utf8(key)).getcstr();
  if (key_data.dptr == NULL) {
    if (logout != NULL) (*logout) << "gdbmclass: out of memory\n";
    return 0;
  }
  key_data.dsize = strlen (key_data.dptr);

  // fetch the result
  return_data = gdbm_fetch (gdbmfile, key_data);
  delete key_data.dptr;

  if (return_data.dptr == NULL) return 0;

  data.setcarr (return_data.dptr, return_data.dsize);
  free (return_data.dptr);
  data = to_uni(data);  // convert to unicode

  return 1;
}


// parses a line of the form <key>value\n
// returns 1 if successful
int gdbmclass::getinfoline (text_t::iterator &here, text_t::iterator end, 
                text_t &key, text_t &value) {
  key.clear();
  value.clear();

  // ignore white space
  while (here != end && is_unicode_space (*here)) here++;

  // get the '<'
  if (here == end || *here != '<') return 0;
  here++;
  
  // get the key
  while (here != end && *here != '>') {
    key.push_back(*here);
    here++;
  }
  
  // get the '>'
  if (here == end || *here != '>') return 0;
  here++;
  
  // get the value
  while (here != end && *here != '\n') {
    value.push_back(*here);
    here++;
  }

  return 1;
}





// a few useful functions

//////////////////////////////////////////////////////////////////////////////////////////
// functions for testing classification strings


// returns 1 if targetdoc is top level of a book (i.e. =~ /B\.\d+$/) - otherwise 0;
int is_top_level (const text_t &targetdoc) {

  text_t::const_iterator here = targetdoc.begin();
  text_t::const_iterator end = targetdoc.end();

  // look for the 'B'
  here = findchar (here, end, 'B');

  // there must be exactly one dot after the 'B'
  if ((here != end) && (countchar (here, end, '.') == 1))
    return 1;

  return 0;
}

// returns 1 if targetdoc is any level of a book (i.e. contains 'B') - otherwise 0 
int is_book (const text_t &targetdoc) {

  text_t::const_iterator here = targetdoc.begin();
  text_t::const_iterator end = targetdoc.end();
  
  here = findchar (here, end, 'B');
  if (here != end) return 1;
  return 0;
}

// returns (in book_top) the top level of the book in targetdoc 
void get_book_top (const text_t &targetdoc, text_t &book_top) {

  text_t::const_iterator here = targetdoc.begin();
  text_t::const_iterator end = targetdoc.end();

  book_top.clear();

  // look for the 'B'
  here = findchar (here, end, 'B');

  // copy up to the second '.'
  int founddot = 0;
  while (here != end) {
    if (*here == '.') {
      if (founddot) return;
      founddot = 1;
    }
    book_top.push_back(*here);
    here++;
  }
}   

// returns (in book) the book section part of the classification
// contained in targetdoc
void get_book (const text_t &targetdoc, text_t &book) {

  text_t::const_iterator here = targetdoc.begin();
  text_t::const_iterator end = targetdoc.end();
  
  book.clear ();

  // look for the 'B'
  here = findchar (here, end, 'B');

  // copy the rest of the string
  while (here != end) {
    book.push_back(*here);
    here ++;
  }
}

// get_parent_section removes the last part from section (i.e.=~ s/\.\d+$//)  
void get_parent_section (text_t &section) {
  int founddot = 0;
  text_t::iterator end;
  while (!founddot && !section.empty()) {
    end = section.end();
    end --;
    if (*end == '.') founddot = 1;
    section.pop_back();
  }
}

// same as above but also returns ths child section that's removed
void get_parent_section (text_t &parentsection, text_t &childsection) {
 int founddot = 0;
 text_t tmp;
 childsection.clear();
  text_t::iterator end;
  while (!founddot && !parentsection.empty()) {
    end = parentsection.end();
    end --;
    if (*end == '.') founddot = 1;
    else tmp.push_back(*end); childsection = tmp + childsection; tmp.clear();
    parentsection.pop_back();
  }
}

// count_dots returns the number of dots ('.') there are
// in a range of a targetdoc string
int count_dots(text_t::const_iterator first, text_t::const_iterator last) {
  return countchar (first, last, '.');
}

int count_dots (const text_t &targetdoc) {
  return count_dots(targetdoc.begin(), targetdoc.end());
}

// returns 1 if targetdoc is a first level descendant
// (i.e. B.n.1, B.n.1.1, B.n.1.1.1 etc.) - otherwise 0
int is_section_top(const text_t &targetdoc) {
  text_t::const_iterator here = targetdoc.begin();
  text_t::const_iterator end = targetdoc.end();
  
  // look for the 'B'
  here = findchar (here, end, 'B');
  here = findchar (here, end, '.');
  if (here != end) here++; // skip over the '.'
  here = findchar (here, end, '.');

  // make sure that all '.' are followed by a '1'
  while (here != end) {
    if (*here != '.') return 0;
    here ++;

    if (here != end) {
      if (*here != '1') return 0;
      here ++;
    }
  }
  return 1;
}

// seperate_parts seperates targetdoc into its classification and booksection
// if classification isn't supplied it gets the first classification for the 
// book from the gdbm
// if booksection doesn't exist it remains blank
void separate_parts(const text_t &targetdoc, gdbmclass &gdbm, 
            text_t &classification, text_t &booksection) {
  
  split_targetdoc (targetdoc, classification, booksection);

  if (classification.empty()) {
    // no classification included so get first one for this book
    gdbm_info info;
    text_t book_top;
    vector<text_t> classarray;
    get_book_top (targetdoc, book_top);
    gdbm.getinfo(book_top, info);
    splitstring (info.classification, classarray);
    if (!classarray.empty()) classification = classarray[0];
    else classification = "C.1";
  }
}

// split_targetdoc splits up a string containing a classification
// and book (or one or the other)
void split_targetdoc(const text_t &targetdoc, text_t &classification, 
             text_t &booksection) {
  
  classification.clear ();
  booksection.clear();

  text_t::const_iterator here = targetdoc.begin();
  text_t::const_iterator end = targetdoc.end();

  // copy everything up to the first 'B'
  while (here != end) {
    if (*here == 'B') break;
    classification.push_back(*here);
    here++;
  }

  // remove middle '.'
  if (!classification.empty() && 
      classification[classification.size()-1] == '.') 
    classification.pop_back();

  // copy the rest of the string
  while (here != end) {
    booksection.push_back(*here);
    here++;
  }
}

// splitstring splits a colon seperated string into an array
void splitstring (const text_t &string, vector<text_t> &array) {
  splitchar (string.begin(), string.end(), ':', array);
}

// get_parents returns the parents array containing all the parents of the
// document specified by classification and booksection
void get_parents (const text_t &targetdoc, vector<text_t> &parents)
{
  text_t::const_iterator here = targetdoc.begin ();
  text_t::const_iterator end = targetdoc.end ();

  text_t currentparent;
  text_t newsuffixpart;
  text_t newsuffix;
  bool first = true;
  while (here != end)
    {
      // if there is a newsuffix add it to the current parent
      // and add that parent to the parents vector
      if (!newsuffix.empty())
    {
      currentparent += newsuffix;
      parents.push_back (currentparent);
    }
      
      // keep getting suffixes until one is found which starts with
      // a number
      newsuffix.clear();
      do 
    {
      here = getdelimitstr (here, end, '.', newsuffixpart);
      if (!first) newsuffix.push_back ('.');
      first = false;
      newsuffix += newsuffixpart;
    } 
      while ((here != end) && !newsuffixpart.empty() && 
         (newsuffixpart[0] < '0' || newsuffixpart[0] > '9'));
    }
}


// get_siblings returns the siblings array containing all the siblings of the current
// classification or booksection
void get_siblings (const text_t &classification, const text_t &booksection, 
           gdbmclass &gdbm, vector<text_t> &siblings) {
  
  gdbm_info info;
  
  if (booksection.empty() &&  classification.size() == 1) {
    // top level classification has no siblings
    return;

  } else if (booksection.empty()) {
    // get classification siblings
    gdbm.getinfo(classification, info);
    gdbm.getinfo(info.parent, info); // info is now parent info
    splitstring(info.contents, siblings);
    return;

  } else {
    // get book section siblings
    if (is_top_level(booksection)) {
      // top level of book so siblings are children of classification
      gdbm.getinfo(classification, info);
      splitstring(info.contents, siblings);

      // add classifications to book sections
      for (unsigned int i = 0; i < siblings.size(); i++) {
    if (is_book(siblings[i])) siblings[i] = classification + "." + siblings[i];
      } 

    } else {
      // siblings come from immediate parent
      gdbm.getinfo(booksection, info);
      gdbm.getinfo(info.parent, info); // info is now parent info
      splitstring(info.contents, siblings);

      // add classifications to book sections
      for (unsigned int i = 0; i < siblings.size(); i++) {
    if (is_book(siblings[i])) siblings[i] = classification + "." + siblings[i];
      } 
    }
  }
}

// compares section 1 and section 2 and returns 1 if section2 belongs to 
// the same chapter as section1 (i.e. is sibling of or child of or child of sibling) 
int are_same_chapter(text_t section1, text_t section2)
{
  get_parent_section(section1);

  while (!section2.empty()) {
    get_parent_section(section2);
    if (section2 == section1) return 1;
  }
  return 0;
}

// get_first_section gets the first section from a colon separated
// list (instring)
void get_first_section(const text_t &instring, text_t &returnstring) {

  returnstring.clear();
  
  text_t::const_iterator here = instring.begin();
  text_t::const_iterator end = instring.end();
  
  while (here != end) {
    if (*here == ':') return;
    returnstring.push_back(*here);
    here ++;
  }
}


// removes html tags from string - everything after < will be removed
// if < occurs without >
void remove_tags (text_t &text) 
{
  text_t::const_iterator here = text.begin ();
  text_t::const_iterator end = text.end ();
  int found = 0;
  text_t tmp;

  while (here != end) {
    if (*here == '<') {found = 1; here ++; continue;}
    if (*here == '>') {found = 0; here ++; continue;}

    if (!found) tmp.push_back(*here);
    here ++;
  }
  text = tmp;
}

// checks text to see if it is a number (i.e. contains only 0-9)
// returns 1 if true, 0 if false
int is_number (text_t &text) {

  text_t::const_iterator here = text.begin();
  text_t::const_iterator end = text.end();

  while (here != end) {
    if ((*here!='0') && (*here!='1') && (*here!='2') &&
    (*here!='3') && (*here!='4') && (*here!='5') &&
    (*here!='6') && (*here!='7') && (*here!='8') &&
    (*here!='9')) return 0;
    here ++;
  }
  return 1;
}

// functions related to sorting

// returns whatever comes after ':#:' in str
// -- this is a nasty hack that I'm sure Rodger will want to change ;-)
text_t get_section_str(const text_t &str) {

  text_t ret;
  int found = 0;

  text_t::const_iterator here = str.begin();
  text_t::const_iterator end = str.end();

  while (here != end) {
    if (found) {
      ret.push_back(*here);
    } else {
      here = findchar (here, end, ':');
      if ((*(here+1) == '#') && (*(here+2) == ':')) {
    found = 1;
    here = here+2;
      }
    }
    here ++;
  }
  return ret;
}

// removes leading spaces and leading 'the' 'a' and 'an' 
// from string
void alphabetize_string_english (text_t &text) {
  
  if (text.empty()) return;

  text_t firstword;
  char *word;

  text_t::iterator here = text.begin();
  text_t::const_iterator end = text.end();

  if ((*here != ' ') && (*here != 'a') && (*here != 'A') &&
      (*here != 't') && (*here != 'T')) return;

  int foundchar = 0;
  while (here != end) {
    if (*here == ' ' && !foundchar) {here ++; continue;}
    if (*here == ' ' && foundchar) {
      text.erase(text.begin(), (here+1));
      break;
    }
    foundchar ++;
    if (foundchar == 1) {
      getdelimitstr (here, end, ' ', firstword);
      word = firstword.getcstr();
      if ((compare_str(word, "the") != 0) && (compare_str(word, "a") != 0) &&
      (compare_str(word, "an") != 0)) break;
    }
    here ++;
  }
  delete word;
}

// removes leading space, puts last name before 
// any preceeding names
void alphabetize_string_name (text_t &text) {
  
  if (text.empty()) return;
  
  text_t lastname;
  char *lname;
  vector<text_t> words;
  splitchar (text.begin(), text.end(), ' ', words);
  lastname = words.back();
  words.pop_back();
  lname = lastname.getcstr();
  
  while ((compare_str(lname, "jnr") == 0) || (compare_str(lname, "snr") == 0) ||
     (compare_str(lname, "esq") == 0)) {
    lastname = words.back();
    words.pop_back();
    lname = lastname.getcstr();
  }

  text.clear();
  joinchar (words, ' ', text);
  text = lastname + text;
}

char ** string_add (char **array, int *len, char *str) {
  char **ret;

  ret = (char**)realloc(array, (*len+1)*sizeof(char*));
  ret[*len] = (char*)strdup(str);
  (*len) ++;

  return ret;
}

void string_sort (char **array, int len) {
  qsort((void*)array, (unsigned int)(len), sizeof(char*), compare_str);
}

void string_free (char **array, int len) {
  for (int i = 0; i < len; i++)
    free (array[i]);
  free (array);
}

// returns a date of form _dec_ 31, 1999 
// input is date of type 19991231
// at least the year must be present in date
text_t format_date (const text_t &date) {
  text_t::const_iterator here = date.begin();
  text_t::const_iterator end = date.end();

  text_t year, month, day, dreturn;
  int i;

  for (i = 0; i < 4 && here != end; i++) {
    year.push_back(*here);
    here ++;
  }
  if (year.empty()) return "";

  for (i = 0; i < 2 && here != end; i++) {
    month.push_back(*here);
    here ++;
  }
  for (i = 0; i < 2 && here != end; i++) {
    day.push_back(*here);
    here ++;
  }

  if (!month.empty()) format_month(month);
  
  if (!day.empty()) format_day(day);
  
  if (!month.empty()) {
    dreturn += month + " ";
    if (!day.empty()) {
      dreturn += day + ", ";
    }
  }
  dreturn += year;
  return dreturn;
}

void format_month (text_t &month) {
  if (month == "01") month = "_jan_";
  else if (month == "02") month = "_feb_"; 
  else if (month == "03") month = "_mar_";
  else if (month == "04") month = "_apr_"; 
  else if (month == "05") month = "_may_";
  else if (month == "06") month = "_jun_";
  else if (month == "07") month = "_jul_";
  else if (month == "08") month = "_aug_"; 
  else if (month == "09") month = "_sep_";
  else if (month == "10") month = "_oct_"; 
  else if (month == "11") month = "_nov_";
  else if (month == "12") month = "_dec_";
  else month.clear();
}
  
void  format_day(text_t &day) {
  if (day[0] == '0') {
    char tmp = day[1];
    day.clear();
    day.push_back(tmp);
  }
}