Changeset 10163

Timestamp:

2005-06-23T11:30:09+12:00 (19 years ago)

Author:

davidb

Message:

lucene_passes.pl upgraded to support incremental building. Changes mostly
involve supporting minus options (i.e. -create to start a new index, nothing
for incremental) and passing this down into the indexer code.

File:

• trunk/gsdl/bin/script/lucene_passes.pl (modified) (5 diffs)

trunk/gsdl/bin/script/lucene_passes.pl

@@ -37 +37 @@
 
 use util;
+use ghtml;
 
 
 sub open_java_lucene
 {
-    my ($doc_tag_level,$full_builddir,$indexdir) = @_;
+    my ($doc_tag_level,$full_builddir,$indexdir,$create,$verbosity) = @_;
 
     my $bin_java = &util::filename_cat($ENV{'GSDLHOME'},"bin","java");
@@ -47 +48 @@
 
     my $java_lucene = "java -classpath \"$classpath\" GS2LuceneIndexer";
-    my $java_cmd = "$java_lucene $doc_tag_level \"$full_builddir\" $indexdir";
+    my $cmd_options = "$create -verbosity $verbosity";
+    my $java_cmd = "$java_lucene $cmd_options $doc_tag_level \"$full_builddir\" $indexdir";
 
     if (!open (PIPEOUT, "| $java_cmd")) {
@@ -74 +76 @@
     print DOCOUT $doc_xml;
     close(DOCOUT);
+
+    my @secs =  ($doc_xml =~ m/<Sec\s+gs2:id="\d+"\s*>.*?<\/Sec>/sg);
+
+    foreach my $sec (@secs) {
+    my ($docnum,$sec_text) = ($sec =~ m/<Sec\s+gs2:id="(\d+)"\s*>(.*?)<\/Sec>/s);
+    my $docnum_filename 
+        = &util::filename_cat($full_textdir,"$docnum.xml");
+    
+    open(SECOUT,">$docnum_filename") 
+        || die "Unable to open $docnum_filename";
+
+    print SECOUT &ghtml::unescape_html($sec_text);
+    close(SECOUT);
+    }
+
 }
 
@@ -126 +143 @@
     my $argc = scalar(@argv);
 
-    if ($argc<4) {
-    print STDERR "Usage: $PROGNAME mode(text|index) doc-tag-level build-dir index-name\n";
+    my $create = "";
+    my $verbosity = 1;
+
+    my @filtered_argv = ();
+
+    my $i = 0;
+    while ($i<$argc) {
+    if ($argv[$i] =~ m/^-(.*)$/) {
+
+        my $option = $1;
+
+        # -create causes build to be incremental
+        if ($option eq ("create")) {
+        $create = "-create";
+        }
+
+        # -verbosity num
+        elsif ($option eq "verbosity") {
+        $i++;
+        if ($i<$argc) {
+            $verbosity = $argv[$i];
+        }
+        }
+        else {
+        print STDERR "Unrecognised minus option: -$option\n";
+        }
+    }
+    else {
+        push(@filtered_argv,$argv[$i]);
+    }
+    $i++;
+    }
+
+    my $filtered_argc = scalar(@filtered_argv);
+
+    if ($filtered_argc < 4) {
+    print STDERR "Usage: $PROGNAME [-create|-verbosity num] \"text\"|\"index\" doc-tag-level build-dir index-name\n";
     exit 1;
     }
 
-    my $mode = $argv[0];
-    my $doc_tag_level = $argv[1];
-    my $full_builddir = $argv[2];
-    my $indexdir      = $argv[3];
-    print STDERR "**** ARGS = ", join(" ", @argv), "\n";
+    my $mode = $filtered_argv[0];
+    my $doc_tag_level = $filtered_argv[1];
+    my $full_builddir = $filtered_argv[2];
+    my $indexdir      = $filtered_argv[3];
+###    print STDERR "**** ARGS = ", join(" ", @argv), "\n";
 
     my $full_textdir = &util::filename_cat($full_builddir,"text");
@@ -141 +193 @@
     if ($mode eq "index") {
     # don't need the lucene stuff if we are just storing the docs
-    open_java_lucene($doc_tag_level,$full_builddir,$indexdir);
+    open_java_lucene($doc_tag_level,$full_builddir,$indexdir,$create,$verbosity);
     }
     monitor_xml_stream($mode, $full_textdir);