Index: 2-extensions/parallel-building/trunk/src/perllib/dbutil/sqlite.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/dbutil/sqlite.pm	(revision 27202)
+++ 	(revision )
@@ -1,369 +1,0 @@
-###########################################################################
-#
-# dbutil::sqlite -- utility functions for writing to sqlite databases
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2009-2010  DL Consulting Ltd.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package dbutil::sqlite;
-
-use strict;
-
-# Please set $db_fast to 1 if you wish to enable faster I/O to the database by using
-# optimisations such as PRAGMA journal_mode (MEMORY instead of DELETE) and synchronous (OFF instead of FULL)
-# Please be aware that in this case it will be less secure and the database file
-# may become corrupted if the if the operating system crashes or the computer loses
-# power before that data has been written to the disk surface.
-# But the speed gain is about 50x
-my $db_fast = 0;
-
-# Set to 1 to enable Write Ahead Logging - which is supposed to allow multiple
-# readers/writers on a SQLite database (overridden by db_fast).
-my $db_wal = 0;
-
-
-# -----------------------------------------------------------------------------
-#   SQLITE IMPLEMENTATION
-# -----------------------------------------------------------------------------
-
-sub open_infodb_write_handle
-{
-  my $infodb_file_path = shift(@_);
-  my $opt_append = shift(@_);
- 
-  my $sqlite3_exe = &util::filename_cat($ENV{'GSDLHOME'},"bin",$ENV{'GSDLOS'}, "sqlite3" . &util::get_os_exe());
-  my $infodb_handle = undef;
-
-  if (!-e "$sqlite3_exe")
-  {
-      print STDERR "Error: Unable to find $sqlite3_exe\n";
-      return undef;
-  } 
-
-  # running sqlite3 with the pragma journal_mode=memory, causes sqlite to print out the 
-  # word "memory". While this is not a problem usually, in our case, this ends up going out
-  # to the web page first, as part of the web page's headers, thus ruining the web page
-  # which causes an Internal Server Error (500). Therefore, we redirect sqlite's output to
-  # the nul device instead. 
-  my $nul_device="";
-  if($db_fast || $db_wal) {
-	if($ENV{'GSDLOS'} =~ m/windows/) {
-		$nul_device=">NUL";
-	} else {
-		$nul_device=">/dev/null"; # linux, mac
-	}
-  }  
-  
-  if(!open($infodb_handle, "| \"$sqlite3_exe\" \"$infodb_file_path\"$nul_device"))
-  {
-      print STDERR "Error: Failed to open pipe to \"$sqlite3_exe\" \"$infodb_file_path\"\n";
-      print STDERR "       $!\n";
-      return undef;
-  }
-
-  binmode($infodb_handle,":utf8");
-  
-   # Add extra optimisations, less secure but with a massive gain in performance with large databases which are often uptaded
-   # They should be set before the transaction begins
-  if (defined $db_fast && $db_fast == 1) {
-	print $infodb_handle "PRAGMA synchronous=OFF;\n";
-	print $infodb_handle "PRAGMA journal_mode=MEMORY;\n";
-  }
-  # Allow parallel readers/writers by using a Write Ahead Logger
-  elsif ($db_wal)
-  {
-    print $infodb_handle "PRAGMA journal_mode=WAL;\n";
-  }
-
-  # This is very important for efficiency, otherwise each command will be actioned one at a time
-  print $infodb_handle "BEGIN TRANSACTION;\n";
-  
-
-
-  if (!(defined $opt_append) || ($opt_append ne "append")) {
-    print $infodb_handle "DROP TABLE IF EXISTS data;\n";
-	print $infodb_handle "DROP TABLE IF EXISTS document_metadata;\n";
-  }
-   
-  print $infodb_handle "CREATE TABLE IF NOT EXISTS data (key TEXT PRIMARY KEY, value TEXT);\n";
-  print $infodb_handle "CREATE TABLE IF NOT EXISTS document_metadata (id INTEGER PRIMARY KEY, docOID TEXT, element TEXT, value TEXT);\n";
-
-  # This is crucial for efficiency when importing large amounts of data
-  print $infodb_handle "CREATE INDEX IF NOT EXISTS dmd ON document_metadata(docOID);\n";
-
-  return $infodb_handle;
-}
-
-
-sub close_infodb_write_handle
-{
-  my $infodb_handle = shift(@_);
-
-  # This is crucial for efficient queries on the database!
-  print $infodb_handle "CREATE INDEX IF NOT EXISTS dme ON document_metadata(element);\n";
-  
-  # Close the transaction we began after opening the file
-  print $infodb_handle "END TRANSACTION;\n";
-
-  close($infodb_handle);
-}
-
-
-sub read_infodb_cmd
-{
-  my $infodb_file_path = shift(@_);
-  my $sqlcmd = shift(@_);
-
-  my $result = "";
-
-  my $sqlite3_exe = &util::filename_cat($ENV{'GSDLHOME'},"bin",$ENV{'GSDLOS'}, "sqlite3" . &util::get_os_exe());
-  my $infodb_handle = undef;
-  my $cmd = "\"$sqlite3_exe\" \"$infodb_file_path\" \"$sqlcmd\"";
-
-  if (!-e "$sqlite3_exe" || !open($infodb_handle, "$cmd |"))
-  {
-      print STDERR "Unable to execute: $cmd\n";
-      print STDERR "$!\n";
-  }
-  else {
-
-      binmode($infodb_handle, ":utf8");
-      my $line;
-      while (defined($line=<$infodb_handle>)) {
-	  $result .= $line;
-      }
-
-      close($infodb_handle);
-  }
-
-  return $result;
-}
-
-sub get_infodb_file_path
-{
-  my $collection_name = shift(@_);
-  my $infodb_directory_path = shift(@_);
-
-  my $infodb_file_extension = ".db";
-  my $infodb_file_name = &util::get_dirsep_tail($collection_name) . $infodb_file_extension;
-  return &util::filename_cat($infodb_directory_path, $infodb_file_name);
-}
-
-
-sub read_infodb_file
-{
-  my $infodb_file_path = shift(@_);
-  my $infodb_map = shift(@_);
-
-
-  my $keys_str = read_infodb_cmd($infodb_file_path,"SELECT key FROM data ORDER BY key;");
-
-  my @keys = split(/\n/,$keys_str);
-
-  foreach my $k (@keys) {
-      
-      my $k_safe = &sqlite_safe($k);
-      my $select_val_cmd = "SELECT value FROM data WHERE key='$k_safe';";
-
-      my $val_str = read_infodb_cmd($infodb_file_path,$select_val_cmd);
-
-      $infodb_map->{$k} = $val_str;
-  }
-
-}
-
-
-sub read_infodb_keys
-{
-  my $infodb_file_path = shift(@_);
-  my $infodb_map = shift(@_);
-
-
-  my $keys_str = read_infodb_cmd($infodb_file_path,"SELECT key FROM data;");
-
-  my @keys = split(/\n/,$keys_str);
-
-  foreach my $key (@keys)
-  {
-      $infodb_map->{$key} = 1;
-  }
-}
-
-sub read_infodb_rawentry
-{
-  my $infodb_file_path = shift(@_);
-  my $infodb_key = shift(@_);
-
-  
-  my $key_safe = &sqlite_safe($infodb_key);
-  my $select_val_cmd = "SELECT value FROM data WHERE key='$key_safe';";
-
-  my $val_str = read_infodb_cmd($infodb_file_path,$select_val_cmd);
-
-  return $val_str
-}
-
-sub read_infodb_entry
-{
-  my $infodb_file_path = shift(@_);
-  my $infodb_key = shift(@_);
-
-  my $val_str = read_infodb_rawentry($infodb_file_path,$infodb_key);
-  
-  my $rec_hash = &dbutil::convert_infodb_string_to_hash($val_str);
-
-  return $rec_hash;
-}
-
-
-sub write_infodb_entry
-{
-  my $infodb_handle = shift(@_);
-  my $infodb_key = shift(@_);
-  my $infodb_map = shift(@_);
-  
-
-  # Add the key -> value mapping into the "data" table
-  my $infodb_entry_value = "";
-  foreach my $infodb_value_key (keys(%$infodb_map))
-  {
-    foreach my $infodb_value (@{$infodb_map->{$infodb_value_key}})
-    {
-      $infodb_entry_value .= "<$infodb_value_key>" . $infodb_value . "\n";
-    }
-  }
-  
-  my $safe_infodb_key = &sqlite_safe($infodb_key);
-  print $infodb_handle "INSERT OR REPLACE INTO data (key, value) VALUES ('" . $safe_infodb_key . "', '" . &sqlite_safe($infodb_entry_value) . "');\n";
-
-  # If this infodb entry is for a document, add all the interesting document metadata to the
-  #   "document_metadata" table (for use by the dynamic classifiers)
-  if ($infodb_key !~ /\./ && $infodb_entry_value =~ /\<doctype\>doc\n/)
-  {
-	
-    print $infodb_handle "DELETE FROM document_metadata WHERE docOID='" . $safe_infodb_key . "';\n";
-	
-    foreach my $infodb_value_key (keys(%$infodb_map))
-    {
-      # We're not interested in most of the automatically added document metadata
-      next if ($infodb_value_key eq "archivedir" ||
-               $infodb_value_key eq "assocfilepath" ||
-               $infodb_value_key eq "childtype" ||
-               $infodb_value_key eq "contains" ||
-               $infodb_value_key eq "docnum" ||
-               $infodb_value_key eq "doctype" ||
-               $infodb_value_key eq "Encoding" ||
-               $infodb_value_key eq "FileSize" ||
-               $infodb_value_key eq "hascover" ||
-               $infodb_value_key eq "hastxt" ||
-               $infodb_value_key eq "lastmodified" ||
-               $infodb_value_key eq "metadataset" ||
-               $infodb_value_key eq "thistype" ||
-               $infodb_value_key =~ /^metadatafreq\-/ ||
-               $infodb_value_key =~ /^metadatalist\-/);
-
-      foreach my $infodb_value (@{$infodb_map->{$infodb_value_key}})
-      {
-		 print $infodb_handle "INSERT INTO document_metadata (docOID, element, value) VALUES ('" . $safe_infodb_key . "', '" . &sqlite_safe($infodb_value_key) . "', '" . &sqlite_safe($infodb_value) . "');\n";
-      }
-    }
-  }
-
-	 #### DEBUGGING
-	#my $new_file = "D:\\sql.txt";
-	#open(FOUT, ">>$new_file") or die "Unable to open $new_file for writing out sql statements...ERROR: $!\n";
-    #print FOUT "BEGIN;\n".$insertStatementsBuffer."\nEND;\n";
-    #close(FOUT);
-	#print STDERR $insertStatementsBuffer; 
-	 #### END DEBUGGING
-}
-
-sub write_infodb_rawentry
-{
-  my $infodb_handle = shift(@_);
-  my $infodb_key = shift(@_);
-  my $infodb_val = shift(@_);
-
-  my $safe_infodb_key = &sqlite_safe($infodb_key);
-  print $infodb_handle "INSERT OR REPLACE INTO data (key, value) VALUES ('" . $safe_infodb_key . "', '" . &sqlite_safe($infodb_val) . "');\n";
-}
-
-
-sub set_infodb_entry
-{
-  my $infodb_file_path = shift(@_);
-  my $infodb_key = shift(@_);
-  my $infodb_map = shift(@_);
-  my $infodb_handle = open_infodb_write_handle($infodb_file_path, "append");
-
-  if (!defined $infodb_handle) {
-      print STDERR "Error: Failed to open infodb write handle\n";
-	  return -1;
-  }
-  else {
-	  write_infodb_entry($infodb_handle,$infodb_key,$infodb_map);
-      close_infodb_write_handle($infodb_handle);
-  }
-  # Not currently checking for errors on write to DB
-  return 0;
-
-}
-
-
-
-sub delete_infodb_entry
-{
-  my $infodb_handle = shift(@_);
-  my $infodb_key = shift(@_);
-
-  # Delete the key from the "data" table
-
-  my $safe_infodb_key = &sqlite_safe($infodb_key);
-
-  print $infodb_handle "DELETE FROM data WHERE key='" . $safe_infodb_key . "';\n";
-
-  # If this infodb entry is for a document, delete the
-  #   "document_metadata" table entry also (for use by the dynamic classifiers)
-  if ($infodb_key !~ /\./)
-  {
-      # Possible for there not to be a docOID matching this infodb_key
-      # (entries are only made when <doctype> == doc
-      # Attempt to delete it, and don't complain if one isn't found
-
-      print $infodb_handle "DELETE FROM document_metadata WHERE docOID='" . $safe_infodb_key . "';\n";
-
-  }
-}
-
-
-sub sqlite_safe
-{
-  my $value = shift(@_);
-
-  # Escape any single quotes in the value
-  $value =~ s/\'/\'\'/g;
-
-  return $value;
-}
-
-
-1;
