Changeset 1679

Timestamp:

2000-11-20T14:03:26+13:00 (23 years ago)

Author:

sjboddie

Message:

Re-Added recent changes that were lost when the CVS repository was moved.
Mostly stuff to do with the collector and building code.

Location:

trunk/gsdl

Files:

• perllib/doc.pm (modified) (2 diffs)
• perllib/mgbuilder.pm (modified) (9 diffs)
• src/recpt/collectoraction.cpp (modified) (15 diffs)
• src/recpt/collectoraction.h (modified) (2 diffs)
• src/recpt/nullproto.cpp (modified) (1 diff)
• src/recpt/nullproto.h (modified) (1 diff)
• src/recpt/recptproto.cpp (modified) (1 diff)
• src/recpt/recptproto.h (modified) (1 diff)
• src/w32server/cgiwrapper.cpp (modified) (3 diffs)

trunk/gsdl/perllib/doc.pm

@@ -220 +220 @@
                        $ENV{'GSDLOS'},"hashfile$osexe");
     my $result = "NULL";
-
+    
     if (-e "$hashfile_exe") {
-    $result = `$hashfile_exe \"$filename\"`;
+#   $result = `\"$hashfile_exe\" \"$filename\"`;
+    $result = `hashfile$osexe \"$filename\"`;
     ($result) = $result =~ /:\s*([0-9a-f]+)/i;
 
@@ -252 +253 @@
 
     } else {
-        
-        print STDERR "doc::set_OID source filename undefined/non-existant (continuing)\n";
+
+        # this warning causes more confusion than it's worth I think
+        # -- sorry Gordon.
+#       print STDERR "doc::set_OID source filename undefined/non-existant (continuing)\n";
         
         $filename = &util::get_tmp_filename();

trunk/gsdl/perllib/mgbuilder.pm

@@ -203 +203 @@
     } else {
     if (!-e "$mg_passes_exe" || 
-        !open (PIPEOUT, "| $mg_passes_exe -f $fulltextprefix -b $maxdocsize -T1 $osextra")) {
+#       !open (PIPEOUT, "| \"$mg_passes_exe\" -f \"$fulltextprefix\" -b $maxdocsize -T1 $osextra")) {
+        !open (PIPEOUT, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T1 $osextra")) {
         die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
     }
@@ -234 +235 @@
         die "mgbuilder::compress_text - couldn't run $mg_compression_dict_exe\n";
     }
-    system ("$mg_compression_dict_exe -f $fulltextprefix -S -H -2 -k 5120 $osextra");
+#   system ("\"$mg_compression_dict_exe\" -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra");
+    system ("mg_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra");
 
     # -b $maxdocsize sets the maximum document size to be 12 meg
     if (!-e "$mg_passes_exe" || 
-        !open ($handle, "| $mg_passes_exe -f $fulltextprefix -b $maxdocsize -T2 $osextra")) {
+#       !open ($handle, "| \"$mg_passes_exe\" -f \"$fulltextprefix\" -b $maxdocsize -T2 $osextra")) {
+        !open ($handle, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T2 $osextra")) {
         die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
     }
@@ -484 +487 @@
     } else {
     if (!-e "$mg_passes_exe" || 
-        !open (PIPEOUT, "| $mg_passes_exe -f $fullindexprefix -b $maxdocsize " .
+#       !open (PIPEOUT, "| \"$mg_passes_exe\" -f \"$fullindexprefix\" -b $maxdocsize " .
+        !open (PIPEOUT, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " .
            "-$index_level -m 32 -s 0 -G -t 10 -N1 $osextra")) {
         die "mgbuilder::build_index - couldn't run $mg_passes_exe\n";
@@ -509 +513 @@
         die "mgbuilder::build_index - couldn't run $mg_perf_hash_build_exe\n";
     }
-    system ("$mg_perf_hash_build_exe -f $fullindexprefix $osextra");
+#   system ("\"$mg_perf_hash_build_exe\" -f \"$fullindexprefix\" $osextra");
+    system ("mg_perf_hash_build$exe -f \"$fullindexprefix\" $osextra");
 
     if (!-e "$mg_passes_exe" || 
-        !open ($handle, "| $mg_passes_exe -f $fullindexprefix -b $maxdocsize " .
+#       !open ($handle, "| \"$mg_passes_exe\" -f \"$fullindexprefix\" -b $maxdocsize " .
+        !open ($handle, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " .
            "-$index_level -c 3 -G -t 10 -N2 $osextra")) {
         die "mgbuilder::build_index - couldn't run $mg_passes_exe\n";
@@ -536 +542 @@
         die "mgbuilder::build_index - couldn't run $mg_weights_build_exe\n";
     }
-    system ("$mg_weights_build_exe -f $fullindexprefix -t $fulltextprefix $osextra");
+#   system ("\"$mg_weights_build_exe\" -f \"$fullindexprefix\" -t \"$fulltextprefix\" $osextra");
+    system ("mg_weights_build$exe -f \"$fullindexprefix\" -t \"$fulltextprefix\" $osextra");
 
     # create 'on-disk' stemmed dictionary
@@ -543 +550 @@
         die "mgbuilder::build_index - couldn't run $mg_invf_dict_exe\n";
     }
-    system ("$mg_invf_dict_exe -f $fullindexprefix $osextra");
+#   system ("\"$mg_invf_dict_exe\" -f \"$fullindexprefix\" $osextra");
+    system ("mg_invf_dict$exe -f \"$fullindexprefix\" $osextra");
 
 
@@ -551 +559 @@
         die "mgbuilder::build_index - couldn't run $mg_stem_idx_exe\n";
     }
-    system ("$mg_stem_idx_exe -b 4096 -s1 -f $fullindexprefix $osextra");
-    system ("$mg_stem_idx_exe -b 4096 -s2 -f $fullindexprefix $osextra");
-    system ("$mg_stem_idx_exe -b 4096 -s3 -f $fullindexprefix $osextra");
+#   system ("\"$mg_stem_idx_exe\" -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
+    system ("mg_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
+#   system ("\"$mg_stem_idx_exe\" -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
+    system ("mg_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
+#   system ("\"$mg_stem_idx_exe\" -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
+    system ("mg_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
 
     
@@ -603 +614 @@
     $handle = STDOUT;
     } else {
-    if (!-e "$txt2db_exe" || !open (PIPEOUT, "| $txt2db_exe $fulldbname")) {
+#   if (!-e "$txt2db_exe" || !open (PIPEOUT, "| \"$txt2db_exe\" \"$fulldbname\"")) {
+    if (!-e "$txt2db_exe" || !open (PIPEOUT, "| txt2db$exe \"$fulldbname\"")) {
         die "mgbuilder::make_infodatabase - couldn't run $txt2db_exe\n";
     }
@@ -679 +691 @@
     my $mgstat_exe = &util::filename_cat($exedir, "mgstat$exe");
     my $input_file = &util::filename_cat ("text", $self->{'collection'});
-    if (!-e "$mgstat_exe" || !open (PIPEIN, "$mgstat_exe -d $self->{'build_dir'} -f $input_file |")) {
+#    if (!-e "$mgstat_exe" || !open (PIPEIN, "\"$mgstat_exe\" -d \"$self->{'build_dir'}\" -f \"$input_file\" |")) {
+    if (!-e "$mgstat_exe" || !open (PIPEIN, "mgstat$exe -d \"$self->{'build_dir'}\" -f \"$input_file\" |")) {
     print $outhandle "Warning: Couldn't open pipe to $mgstat_exe to get additional stats\n";
     } else {

trunk/gsdl/src/recpt/collectoraction.cpp

@@ -48 +48 @@
   gsdlosc = NULL;
   gsdlhomec = NULL;
+  pathc = NULL;
 
   cgiarginfo arg_ainfo;
@@ -217 +218 @@
   if (gsdlosc != NULL) delete gsdlosc;
   if (gsdlhomec != NULL) delete gsdlhomec;
+  if (pathc != NULL) delete pathc;
 }
 
 bool collectoraction::init (ostream & /*logout*/) {
 
-  // set up GSDLOS and GSDLHOME environment variables
-  text_t gsdlos;
+  // set up GSDLOS, GSDLHOME and PATH environment variables
+  text_t gsdlos, path;
+  unsigned int path_separator = ':';
 #if defined (__WIN32__)
   gsdlos = "windows";
+  path_separator = ';';
 #else
   utsname *buf = new utsname();
@@ -233 +237 @@
   lc (gsdlos);
 #endif
-  
+
+  pathc = getenv ("PATH");
+  path = filename_cat (gsdlhome, "bin", gsdlos);
+  path.push_back (path_separator);
+  path += filename_cat (gsdlhome, "bin", "script");
+  if (pathc != NULL) {
+    path.push_back (path_separator);
+    path += pathc;
+  }
+  path = "PATH=" + path;
+
   gsdlos = "GSDLOS=" + gsdlos;
   text_t setgsdlhome = "GSDLHOME=" + gsdlhome;
@@ -240 +254 @@
   gsdlosc = gsdlos.getcstr();
   gsdlhomec = setgsdlhome.getcstr();
+  pathc = path.getcstr();
   
   putenv (gsdlosc);
   putenv (gsdlhomec);
+  putenv (pathc);
 
   return true;
@@ -272 +288 @@
     // partially built collection laying about -- may need to think about doing
     // this in the background).
-    text_t cleantmp_cmd = "perl " + filename_cat (gsdlhome, "bin", "script", "cleantmp.pl");
-    char *cleantmp_cmdc = cleantmp_cmd.getcstr();
-    system (cleantmp_cmdc);
-    delete cleantmp_cmdc;
+    gsdl_call_perl ("cleantmp.pl", NULL);
   }
 
@@ -798 +811 @@
   // _header_ -- may be set for pages that require it
   // _faillog_ - set to last 6 lines of .bld file if build failed
+  // _gsdlhome_ - the gsdlhome path (dm_safe)
 
   text_t &collector_page = args["cp"];
@@ -930 +944 @@
       here ++;
     }
-    disp.setmacro ("faillog", "collector", faillog);
+    disp.setmacro ("faillog", "collector", dm_safe(faillog));
   }
 
@@ -942 +956 @@
   if (collector_page == "bildstatus")
     set_statusline (disp, args, logout);
+
+  disp.setmacro ("gsdlhome", "collector", dm_safe(gsdlhome));
 }
 
@@ -974 +990 @@
     
     } else {
-    
-      text_t delete_cmd = "perl " + filename_cat (gsdlhome, "bin", "script", "delcol.pl");
-      delete_cmd += " -f " + collection;
-      char *delete_cmdc = delete_cmd.getcstr();
-      int rv = system (delete_cmdc);
-      delete delete_cmdc;
+      // first we need to free up the collection's collection server
+      // we must do this for the local library (and I guess when using
+      // fastcgi too) as you can't delete the gdbm file while it's
+      // being kept open by the collection server
+      remove_colservr (collection, logout);
+
+      char *collectionc = collection.getcstr();
+      int rv = gsdl_call_perl ("delcol.pl", "-f", collectionc, NULL);
+      delete collectionc;
       if (rv != 0) {
     // deletion failed -- permissions?
@@ -1152 +1171 @@
   if (file_exists (createfile)) {
     return;
+  }
+
+  // set up options
+  text_t options = "-creator \"" + args["bc1contactemail"] + "\"";
+  options += " -title \"" + args["bc1fullname"] + "\"";
+  options += " -about \"" + carriage_replace (args["bc1aboutdesc"], 0) + "\"";
+  options += " -collectdir \"" + tmpdir + "\" ";
+  text_t optionfile = filename_cat (tmpdir, "mkcol.opt");
+  char *optionfilec = optionfile.getcstr();
+  ofstream ofile_out (optionfilec);
+  if (!ofile_out) {
+    message = "tmpfail";
+    delete optionfilec;
+    return;
+  }
+  outconvertclass text_t2ascii;
+  ofile_out << text_t2ascii << options << "\n";
+  ofile_out.close();
+
+  char *collectionc = collection.getcstr();
+
+  // this is just an ordinary system call running in the foreground. we're assuming (hoping??)
+  // that mkcol.pl will run through fast enough that nothing more elaborate is required.
+  gsdl_call_perl ("mkcol.pl", "-optionfile", optionfilec, collectionc, NULL);
+
+  delete optionfilec;
+  delete collectionc;
+
+  // make sure it went ok
+  text_t cfgfile = filename_cat (tmpdir, collection, "etc", "collect.cfg");
+  if (!file_writable (cfgfile)) {
+    message = "mkcolfail";
   } else {
     // create the .create file (this file is just a place holder to let any future
@@ -1165 +1216 @@
       return;
     }
-  }
-
-  // set up options
-  text_t options = "-creator \"" + args["bc1contactemail"] + "\"";
-  options += " -title \"" + args["bc1fullname"] + "\"";
-  options += " -about \"" + carriage_replace (args["bc1aboutdesc"], 0) + "\"";
-  options += " -collectdir \"" + tmpdir + "\" ";
-  text_t optionfile = filename_cat (tmpdir, "mkcol.opt");
-  char *optionfilec = optionfile.getcstr();
-  ofstream ofile_out (optionfilec);
-  delete optionfilec;
-  if (!ofile_out) {
-    message = "tmpfail";
-    return;
-  }
-  outconvertclass text_t2ascii;
-  ofile_out << text_t2ascii << options << "\n";
-  ofile_out.close();
-  
-  // set up the mkcol.pl command
-  text_t mkcol_cmd = "perl ";
-  mkcol_cmd += filename_cat (gsdlhome, "bin", "script", "mkcol.pl");
-  mkcol_cmd += " -optionfile \"" + optionfile + "\" " + collection;
-  char *mkcol_cmdc = mkcol_cmd.getcstr();
-  
-  // just an ordinary system call running in the foreground. we're assuming (hoping??)
-  // that mkcol.pl will run through fast enough that nothing more elaborate is required.
-  system (mkcol_cmdc);
-  delete mkcol_cmdc;
-  
-  // make sure it went ok
-  text_t cfgfile = filename_cat (tmpdir, collection, "etc", "collect.cfg");
-  if (!file_writable (cfgfile)) {
-    message = "mkcolfail";
   }
 }
@@ -1255 +1272 @@
     options += " -download \"" + args["bc1inputdir4"] + "\"";
   text_t optionfile = filename_cat (tmpdir, "build.opt");
-  char *optionfilet = optionfile.getcstr();
-  ofstream ofile_out (optionfilet);
-  delete optionfilet;
+  char *optionfilec = optionfile.getcstr();
+  ofstream ofile_out (optionfilec);
+  delete optionfilec;
   if (!ofile_out) {
     message = "tmpfail";
@@ -1267 +1284 @@
 
   // set up the build command
-  text_t build_cmd = "perl " + filename_cat (gsdlhome, "bin", "script", "build");
-  build_cmd += " -optionfile \"" + optionfile + "\" " + collection;
-#if !defined (__WIN32__)
-  // run in background on unix systems
-  //  build_cmd += " &";
-#endif
+  // Can't seem to get "perl -S" to work properly here on windows 95...
+  //text_t build_cmd = "perl -S build -optionfile \"" + optionfile + "\" " + collection;
+  text_t build_cmd = "perl \"" + filename_cat(gsdlhome, "bin", "script", "build") + 
+    "\" -optionfile \"" + optionfile + "\" " + collection;
   char *build_cmdc = build_cmd.getcstr();
-  //#if defined (__WIN32__)
+  // run build command in background
   gsdl_system (build_cmdc, logout);
-  //#else
-  //  system (build_cmdc);
-  //#endif
   delete build_cmdc;
 }
@@ -1286 +1298 @@
   // c++ code. I ran into some problems though (like how do you write a portable
   // "rm -r" in c++?). One day I'll spend some time sorting it out ... maybe.
-  text_t cancel_cmd = "perl " + filename_cat (gsdlhome, "bin", "script", "cancel_build.pl");
-  cancel_cmd += " -collectdir \"" + filename_cat (gsdlhome, "tmp", args["bc1tmp"]) + 
-    "\" " + args["bc1dirname"];
-  char *cancel_cmdc = cancel_cmd.getcstr();
-  system (cancel_cmdc);
-  delete (cancel_cmdc);
+  text_t collectdir = filename_cat (gsdlhome, "tmp", args["bc1tmp"]);
+  char *collectdirc = collectdir.getcstr();
+  char *collectionc = args["bc1dirname"].getcstr();
+
+  gsdl_call_perl ("cancel_build.pl", "-collectdir", collectdirc, collectionc, NULL);
+  delete (collectdirc);
+  delete (collectionc);
 }
 
@@ -1375 +1388 @@
   logout << "collectoraction::create_colserver: no valid nullproto found\n";
 }
+
+// delete a collection server from the null protocol
+void collectoraction::remove_colservr (const text_t &collection, ostream &logout) {
+
+  recptprotolistclass *protos = recpt->get_recptprotolist_ptr();
+  recptprotolistclass::iterator rprotolist_here = protos->begin();
+  recptprotolistclass::iterator rprotolist_end = protos->end();
+  while (rprotolist_here != rprotolist_end) {
+    if ((*rprotolist_here).p != NULL) {
+      if ((*rprotolist_here).p->get_protocol_name () == "nullproto") {
+    (*rprotolist_here).p->remove_collection (collection, logout);
+    return;
+      }
+    }
+    rprotolist_here ++;
+  }
+
+  logout << "collectoraction::create_colserver: no valid nullproto found\n";
+}

trunk/gsdl/src/recpt/collectoraction.h

@@ -38 +38 @@
   char *gsdlosc;
   char *gsdlhomec;
+  char *pathc;
 
   text_t message;
@@ -79 +80 @@
   void create_colserver (const text_t &collection, ostream &logout);
 
+  void remove_colservr (const text_t &collection, ostream &logout);
+
 public:
   collectoraction ();

trunk/gsdl/src/recpt/nullproto.cpp

@@ -151 +151 @@
 }
 
+// remove_collection deletes the collection server of collection.
+// This only needs to be called if a collectionserver is to be
+// removed while the library is running. The destructor function
+// cleans up all collectservers when the program exits.
+void nullproto::remove_collection (const text_t &collection, ostream &logout) {
+  collectservermapclass::iterator here = cservers.begin();
+  collectservermapclass::iterator end = cservers.end();
+
+  while (here != end) {
+    if ((*here).second.c != NULL && (*here).first == collection) {
+      delete (*here).second.c;
+      cservers.erase (here);
+      return;
+    }
+    here ++;
+  }
+  outconvertclass text_t2ascii;
+  logout << text_t2ascii << "nullproto::remove_collection: failed to remove collectserver for "
+     << collection << "\n";
+}
+
 // this configure will configure each of the collection servers
 void nullproto::configure (const text_t &key, const text_tarray &cfgline) {

trunk/gsdl/src/recpt/nullproto.h

@@ -49 +49 @@
     {cservers.addcollectserver(thecserver);}
 
+  // remove_collection deletes the collection server of collection.
+  // This only needs to be called if a collectionserver is to be
+  // removed while the library is running as the destructor function
+  // cleans up all collectservers when the program exits.
+  void remove_collection (const text_t &collection, ostream &logout);
+
   // this configure will configure each of the collection servers
   void configure (const text_t &key, const text_tarray &cfgline);

trunk/gsdl/src/recpt/recptproto.cpp

@@ -31 +31 @@
 void recptproto::add_collection (const text_t &/*collection*/, void * /*recpt*/, 
                  const text_t &/*gsdlhome*/, const text_t &/*gdbmhome*/) {
+}
+
+void recptproto::remove_collection (const text_t &/*collection*/, ostream &/*logout*/) {
 }
 

trunk/gsdl/src/recpt/recptproto.h

@@ -57 +57 @@
   virtual void add_collection (const text_t &collection, void *recpt, 
                    const text_t &gsdlhome, const text_t &gdbmhome);
+
+  virtual void remove_collection (const text_t &collection, ostream &logout);
 
   // configure should be called for each line in the configuration file

trunk/gsdl/src/w32server/cgiwrapper.cpp

@@ -659 +659 @@
                   RequestInfoT *RequestInfo,
                   RequestFieldsT *RequestFields) {
-  char tmpstr[1024];
-  char tailstr[MAX_URL_SIZE];
+  char tmpstr[MAX_URL_SIZE+1024];
+  char tailstr[MAX_URL_SIZE+16];
   
   // do any url adjustments necessary
@@ -722 +722 @@
 }
 
-int ExamineURIStr(char *URIStr,RequestInfoT *RequestInfo,
+int ExamineURIStr(char *URIStr, RequestInfoT *RequestInfo,
           RequestFieldsT *RequestFields)
 {
@@ -740 +740 @@
       }
       URIStr[len] = '\0';
+    } else {
+      MessageBox (NULL, "POST data too long", "Greenstone Digital Library Software", MB_OK);
     }
   }