source: indexers/trunk/mgpp/text/mgpp_perf_hash_build.cpp@ 19822

/**************************************************************************
 *
 * mgpp_perf_hash_build.cpp -- Program to build a perfect hash function
 * Copyright (C) 1999  Rodger McNab
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 **************************************************************************/

#define _XOPEN_SOURCE 1
// This was added for Solaris, but it makes things worse on Solaris for me...
// #define _XOPEN_SOURCE_EXTENDED 1

// need this to avoid bizarre compiler problems under VC++ 6.0
#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
# include <iostream>
#endif

/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
#if defined (__WIN32__) || defined (__CYGWIN__)
# include "getopt_old.h"
#else
# include <unistd.h>
#endif

#include "sysfuncs.h"
#include "memlib.h"
#include "messages.h"
#include "local_strings.h"
#include "perf_hash.h"
#include "netorder.h"

#include "mg_files.h"
#include "invf.h"
#include "locallib.h"
#include "words.h"
#include "mg.h"

#define POOL_SIZE 1024*1024

static void ProcessFiles (char *filename, int r) {
  FILE *dictFile, *hashFile;
  unsigned long i;
  invf_dict_header idh;
  perf_hash_data *phd;
  u_char *pool;
  unsigned long pool_left;
  u_char **starts;

  // read in the dictionary
  dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
            MAGIC_STEM_BUILD, MG_ABORT);
  if (dictFile==NULL) {
    FatalError(1, "unable to open file");
  }
  idh.Read (dictFile);

  //cerr << idh.lookback<<" "<<idh.word_dict_start<<endl;
  // go to the start of the word dictionary
  fseek (dictFile, idh.word_dict_start, SEEK_SET);
  
  if (!(pool = (u_char *) Xmalloc (POOL_SIZE)))
    FatalError (1, "Out of memory");
  pool_left = POOL_SIZE;

  if (!(starts = (u_char **) Xmalloc (sizeof (u_char *) * idh.word_dict_size)))
    FatalError (1, "Out of memory");
  //cerr << "size= "<< idh.word_dict_size<<endl;
  word_dict_el wordEl;
  wordEl.SetNumLevels (idh.num_levels);
  for (i = 0; i < idh.word_dict_size; ++i) {
    // read the next word and associated information
    wordEl.Read (dictFile, idh.num_levels);

    // push string onto pool data
    register unsigned long l;
    l = wordEl.el.size() + 1;
    if (pool_left < l) {
      pool = (u_char *) Xmalloc (POOL_SIZE);
      pool_left = POOL_SIZE;
    }
    starts[i] = pool;
    
    *pool++ = wordEl.el.size();
    memcpy ((char *) pool, &(wordEl.el[0]), wordEl.el.size());
    //cerr << pool<<"   " <<starts[i]<<endl;
    pool += wordEl.el.size();
    pool_left -= l;
    
  }
  fclose (dictFile);
  //cerr << pool<<"   " <<starts[i-1]<<endl;
  //cerr<<"starts  "<<starts[113529]<<endl;
  //cerr << starts[17][1] << " "<<starts[25][4]<<endl;
  // create perfect hash file
  hashFile = create_file (filename, INVF_DICT_HASH_SUFFIX, "wb",
              MAGIC_HASH, MG_ABORT);
  if (!(phd = gen_hash_func (idh.word_dict_size, starts, r)))
    FatalError (1, "Unable to generate hash function");
  if (write_perf_hash_data (hashFile, phd) == -1)
    FatalError (1, "Unable to write hash function");
  fclose (hashFile);
}



int main (int argc, char **argv) {
  int r = -1;
  char *filename = "";
  int ch;
  msg_prefix = argv[0];
  opterr = 0;

  while ((ch = getopt (argc, argv, "f:d:r:h")) != -1) {
    switch (ch) {
    case 'f':       // input file
      filename = optarg;
      break;
    case 'd':
      set_basepath (optarg);
      break;
    case 'r':
      r = atoi (optarg);
      break;
    case 'h':
    case '?':
      fprintf (stderr, "usage: %s [-f input_file]"
           "[-d data directory] [-r random seed] [-h]\n", argv[0]);
      exit (1);
    }
  }

  ProcessFiles (filename, r);
  return 0;
}