source: trunk/gsdl3/bin/script/convert_coll_from_gs2.pl@ 10360

#!/usr/bin/perl -w

BEGIN {
    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
    die "GSDL3HOME not set\n" unless defined $ENV{'GSDL3HOME'};
    die "GSDL3SRCHOME not set\n" unless defined $ENV{'GSDL3SRCHOME'};
    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
    unshift (@INC, "$ENV{'GSDL3SRCHOME'}/lib/perl/cpan");
}

use colcfg;
use util;
use parsargv;
use FileHandle;
use XML::Writer;
use GDBM_File;

use strict;

&main();
sub print_usage() {
    print STDOUT "Usage: convert_coll_from_gs2.pl [options] coll-name\n";
    print STDOUT "options:\n";
    
    print STDOUT "   -collectdir         Directory where collection lives.\n";
    print STDOUT "   -verbosity          Controls the amount of output.\n";
    print STDOUT "   -defaultlang        The language that is considered the default (for display text etc). defaults to 'en'\n\n";
}

sub main {

    my ($defaultlang, $verbosity, $collectdir);
    # note that no defaults are passed for most options as they're set
    # later (after we check the collect.cfg file)
    if (!&parsargv::parse(\@ARGV,
              'verbosity/\d+/', \$verbosity,
              'collectdir/.*/', \$collectdir,
              'defaultlang/.*/', \$defaultlang)) {
    &print_usage();
    die "\n";
    }

    # get and check the collection name
    my ($collection) = @ARGV;
    if (!defined($collection) || $collection eq "") {
    die "No collection specified\n";
    }
    if ($collection eq "gs2model") {
    die "You cant convert the model collection\n";
    }
    
    if (!defined $collectdir || $collectdir eq "") {
    $collectdir = &util::filename_cat ($ENV{'GSDLHOME'}, "collect");
    }

    if (!defined $defaultlang || $defaultlang eq "") {
    $defaultlang = 'en';
    }
    # add on the coll name
    $collectdir = &util::filename_cat ($collectdir, $collection);
    
    my $collconfigfilename = &util::filename_cat ($collectdir, "etc", "collect.cfg");
    print STDOUT "coll config=$collconfigfilename\n";
    my $collectcfg;
    if (-e $collconfigfilename) {
    $collectcfg = &colcfg::read_collect_cfg ($collconfigfilename);

    } else {
    print STDERR "collect.cfg not found!!";
    die "\n";
    }
    

    my $buildconfigfilename = &util::filename_cat ($collectdir, "index", "build.cfg");
    my $buildcfg;
    if (-e $buildconfigfilename) {
    $buildcfg = &colcfg::read_build_cfg ($buildconfigfilename);

    } else {
    print STDERR "build.cfg not found!!";
    die "\n";
    }
    

     
    my $colloutfile = &util::filename_cat ($collectdir, "etc", "collectionConfig.xml");
    if (-e $colloutfile) {
    print STDOUT "collectionConfig file already exists! overwriting it!\n";
    
    }
   
    my $buildoutfile = &util::filename_cat ($collectdir, "index", "buildConfig.xml");
    if (-e $buildoutfile) {
    print STDOUT "buildConfig file already exists! overwriting it!\n";
    
    }

    my $db_file = &util::filename_cat ($collectdir, "index", "text", "$collection.ldb");
    my $database;
    if (-e $db_file) {
    $database = &open_database($db_file);
    } else {
    print STDERR "gdbm database file $db_file not found!!";
    die "\n";
    }
    
    my $buildtype;
    if (defined $buildcfg->{'buildtype'}) {
    $buildtype = $buildcfg->{'buildtype'};
    } else {
    $buildtype = 'mg';
    }
    
    my $buildoutput = new IO::File(">$buildoutfile");
    my $buildwriter = new XML::Writer(OUTPUT => $buildoutput, NEWLINES => 1);
    
    $buildwriter->startTag('buildConfig', 'xmlns:gsf'=>"http://www.greenstone.org/greenstone3/schema/ConfigFormat");
    
    my $colloutput = new IO::File(">$colloutfile");
    my $collwriter = new XML::Writer(OUTPUT => $colloutput, NEWLINES => 1);
    
    $collwriter->startTag('collectionConfig', 'xmlns:gsf'=>"http://www.greenstone.org/greenstone3/schema/ConfigFormat", 'xmlns:xsl'=>'http://www.w3.org/1999/XSL/Transform');
    
    #output the collection metadata to the collectionConfig file
    $collwriter->startTag('metadataList');
    
    my $creator = $collectcfg->{'creator'};
    &output_metadata($collwriter, 'creator', $creator);
    $collwriter->endTag('metadataList');

    #output the display collectionmeta to collectionConfig.xml
    
    my $collectionmeta = $collectcfg->{'collectionmeta'};
    if (defined $collectionmeta) {
    my %name_map = ('collectionname', 'name',
             'collectionextra', 'description',
             'iconcollection', 'icon',
             'iconcollectionsmall', 'smallicon');
    
    $collwriter->startTag('displayItemList');
    foreach my $entry ( keys %$collectionmeta) {
        if (defined $name_map{$entry}) {
        my $name= $name_map{$entry};
        foreach my $lang (keys %{$collectionmeta->{$entry}}) {
            my $value = $collectionmeta->{$entry}->{$lang};
            if ($entry =~ /^icon/) {
            $value = format_icon_value($value);
            }
            &output_display($collwriter, $name, $lang, $value);
        }
        }
    }
    $collwriter->endTag('displayItemList');
    } 
    
    # output building metadata to build config file 
    my $numdocs = $buildcfg->{'numdocs'};
    $buildwriter->startTag('metadataList');
    &output_metadata($buildwriter, 'numDocs', $numdocs);
    &output_metadata($buildwriter, 'buildType', $buildtype);
    $buildwriter->endTag('metadataList');
    
    
    #indexes
    my $indexmap = {};
    my $firstindex = "";
    my $first = 1;
    if (defined $buildcfg->{'indexmap'}) {
    my $indexmap_t = $buildcfg->{'indexmap'};
    foreach my $i (@$indexmap_t) {
        my ($k, $v) = $i =~ /^(.*)\-\>(.*)$/;
        $indexmap->{$k} = $v;
        if ($first) {
        $firstindex = $v;
        $first = 0;
        }   
    }
    } else {
    print STDERR "indexmap not defined";
    }
    my $defaultindex;
    if (defined $collectcfg->{'defaultindex'}) {
    $defaultindex = $collectcfg->{'defaultindex'};
    $defaultindex = $indexmap->{$defaultindex};
    } else {
    # use the first one
    $defaultindex = $firstindex;
    }
    
    # format stuff
    my $format = $collectcfg->{'format'};

    #output the search stuff to coll cfg
    $collwriter->startTag('search');
    foreach my $i (keys %$indexmap) {
    my $shortname = $indexmap->{$i};
    $collwriter->startTag('index', 'name'=>$shortname);
    #find the coll meta stuff
    my $indexdisplay = ".$i";
    foreach my $lang (keys %{$collectionmeta->{$indexdisplay}}) {
        my $value = $collectionmeta->{$indexdisplay}->{$lang};
        output_display($collwriter, 'name', $lang, $value);
    }
    $collwriter->endTag('index');
    
    }

    # add in the format stuff
    if (defined $format->{'SearchVList'}) {
    
    $collwriter->startTag('format');
    write_format($collwriter, $format->{'SearchVList'}, "document");
    $collwriter->endTag('format');
    }
    
    $collwriter->endTag('search');
    
    $buildwriter->startTag('serviceRackList');
    
    my @levels = ();
    my $defaultlevel;
    
    #do the retrieve service
    # assume mgpp or mg 
    if ($buildtype eq 'mgpp') {
    #for each level
    if (defined $buildcfg->{'indexlevels'}) {
        push @levels, @{$buildcfg->{'indexlevels'}};
        
        if (defined $buildcfg->{'textlevel'}) {
        $defaultlevel = $buildcfg->{'textlevel'};
        } else {
        $defaultlevel = $levels[0];
        }
    } else { #use levels from collect.cfg - must be an old collection
        @levels = ('Document');
        $defaultlevel = 'Document';
        if (defined $collectcfg->{'levels'}) {
        foreach my $l (@{$collectcfg->{'levels'}}) {
            if ($l eq "Section") {
            $defaultlevel = 'Section';
            }
            push @levels, $l;
        }
        }
    }
    
    $buildwriter->startTag('serviceRack', 'name'=>'GS2MGPPRetrieve');
    $buildwriter->emptyTag('defaultLevel', 'name'=>$defaultlevel);

    
    } else {
    $buildwriter->startTag('serviceRack', 'name'=>'GS2MGRetrieve');
    $buildwriter->emptyTag('defaultIndex', 'name'=>$defaultindex);
    
    }
    # close off the Retrieve service
    $buildwriter->endTag('serviceRack');

    # add in the classifiers if needed

    my $count = 1;
    my $phind = 0;
    my $started_classifiers = 0;
    if (defined $collectcfg->{'classify'}) { 
    $collwriter->startTag('browse');
    # add in default format if necessary
    if (defined $format->{"VList"} || defined $format->{"HList"}) {
        # global formats
        $collwriter->startTag('format');
        if (defined $format->{"VList"}) {
        # VLIst applies to both classifier and doc nodes
        write_format($collwriter, $format->{"VList"}, "document");
        write_format($collwriter, $format->{"VList"}, "classifier");
        }
        if (defined $format->{"HList"}) {
        # hlist is only for classifier nodes
        write_format($collwriter, $format->{"HList"}, "horizontal");
        }
        $collwriter->endTag('format');
    }
    my $classifiers = $collectcfg->{'classify'};
    foreach my $cl (@$classifiers) {
        my $name = "CL$count";
        $count++;
        my ($classname) = @$cl[0];
        if ($classname =~ /^phind$/i) {
        $phind=1;
        #should add it into coll config classifiers
        next;
        }
        
        my $horizontalAtTop = &isHorizontalClassifier($database, $name);
        if (not $started_classifiers) {
        $buildwriter->startTag('serviceRack', 'name'=>'GS2Browse');
        $buildwriter->startTag('classifierList');       
        $started_classifiers = 1;
        }
        my $content = ''; #use buttonname first, then metadata
        if ($classname eq "DateList") {
        $content = "Date";
        } else {
        for (my $i=0; $i<scalar(@$cl); $i++) {
            my $arg = @$cl[$i];
            if ($arg eq "-buttonname"){
            $content = @$cl[$i+1];
            last;
            } elsif ($arg eq "-metadata") {
            $content = @$cl[$i+1];
            }
            
        }
        }
        if ($horizontalAtTop) {
        $buildwriter->emptyTag('classifier', 'name'=>$name, 'content'=>$content, 'horizontalAtTop'=>'true');
        } else {
        $buildwriter->emptyTag('classifier', 'name'=>$name, 'content'=>$content);
        } 
        
        
        $collwriter->startTag('classifier', 'name'=>$name);
        my $vlist = $name."VList";
        my $hlist = $name."HList";
        my $dlist = "";
        if ($classname eq "DateList") {
        $dlist = "DateList";
        }
        # need to work out how to split into classifier and document
        if (defined $format->{$vlist} || defined $format->{$hlist} || defined $format->{$dlist}) {
        $collwriter->startTag('format');
        if (defined $format->{$vlist}) {
            write_format($collwriter, $format->{$vlist}, "document");
            write_format($collwriter, $format->{$vlist}, "classifier");
        } 
        if (defined $format->{$hlist}) {
            write_format($collwriter, $format->{$hlist}, "horizontal");
        } 
        
        if (defined $format->{$dlist}) {
            write_format($collwriter, $format->{$dlist}, "document");
        }
        $collwriter->endTag('format');
        }
        $collwriter->endTag('classifier');
    } #foreach classifier
    if ($started_classifiers) {
        # end the classifiers
        $buildwriter->endTag('classifierList');
        # close off the Browse service
        $buildwriter->endTag('serviceRack');
    }
    
    $collwriter->endTag('browse');
    }
    
    
    # the phind classifier is a separate service
    if ($phind) {
    # if phind classifier
    $buildwriter->emptyTag('serviceRack', 'name'=>'PhindPhraseBrowse');
    }
   
    # do the search service
    if ($buildtype eq 'mgpp') {
        
    $buildwriter->startTag('serviceRack', 'name'=>'GS2MGPPSearch');
    $buildwriter->emptyTag('defaultLevel', 'name'=>$defaultlevel);
    $buildwriter->startTag('levelList');
    foreach my $level (@levels) {
        $buildwriter->emptyTag('level', 'name'=>$level);
    }
    $buildwriter->endTag('levelList');
    
    #fieldlist
    my $fieldmap = {};
    my $fieldlist = ();
    if (defined $buildcfg->{'indexfields'}) {
        push @$fieldlist, @{$buildcfg->{'indexfields'}};
        if (defined $buildcfg->{'indexfieldmap'}) {
        my $fieldmap_t = $buildcfg->{'indexfieldmap'};
        foreach my $f (@$fieldmap_t) {
            my ($k, $v) = $f =~ /^(.*)\-\>(.*)$/;
            $fieldmap->{$k} = $v;
        }
        }
        $buildwriter->startTag('fieldList');
        foreach my $f (@$fieldlist) {
        my $field = $fieldmap->{$f};
        $buildwriter->emptyTag('field', 'shortname'=>$field, 'name'=>$f);
        }   
        $buildwriter->endTag('fieldList');
    } else {
        print STDERR "indexfieldmap not defined";
    }

    # do the search types if there
    if (defined $collectcfg->{'searchtype'}) {
        $buildwriter->startTag('searchTypeList');
        foreach my $st (@{$collectcfg->{'searchtype'}}) {
        $buildwriter->emptyTag('searchType', 'name'=>$st);
        }
        $buildwriter->endTag('searchTypeList');
    }
    } elsif ($buildtype eq 'mg') {
    $buildwriter->startTag('serviceRack', 'name'=>'GS2MGSearch');
    }
    
    
    $buildwriter->emptyTag('defaultIndex', 'name'=>$defaultindex);
    $buildwriter->startTag('indexList');
    #for each index
    foreach my $i (keys %$indexmap) {
    my $index = $indexmap->{$i};
    $buildwriter->emptyTag('index', 'name'=>$index);
    }   
    $buildwriter->endTag('indexList');
    
    $buildwriter->endTag('serviceRack');
    
    $buildwriter->endTag('serviceRackList');
    $buildwriter->endTag('buildConfig');
    $collwriter->endTag('collectionConfig');
    $collwriter->end();
    $buildwriter->end();
    $buildoutput->close();
    $colloutput->close();
    &close_database($database);
}


sub output_metadata {
    my ($writer, $metaname,  $metavalue) = @_;
    $writer->startTag('metadata',  'name'=>$metaname);
    $writer->characters($metavalue);
    $writer->endTag('metadata');
}

sub output_display {
    my ($writer, $name, $lang, $value) = @_;
    $lang = 'en' if $lang eq 'default';
    if ($lang =~ /^\[/) {
    ($lang) = $lang =~ /\[l=(.*)\]/;
    }
    $writer->startTag('displayItem', 'name'=>$name, 'lang'=>$lang);
    $writer->characters($value);
    $writer->endTag('displayItem');
}
sub format_icon_value {
    my ($value) = @_;
    if ($value =~ /^_/) {
    my ($newvalue) = $value =~ /\/([^\/]*)$/;
    if ($newvalue) {
        return $newvalue;
    }
    }
    return $value;
}

sub write_format {
    my ($writer, $old_format, $node_type) = @_;
    # replace \' with '
    $old_format =~ s/\\\'/\'/g;

    #convert [] to <gsf:...>
    # assume no nesting {If} or {Or} for now
    #remove IFs, for now just make the first option true
    #$old_format =~ s/\{If\}\{[^,]*,([^,\}]*)(,[^\}]*)?\}/$1/g;
    #remove ORs, for now just make the first option true
    $old_format =~ s/\{If\}\{([^\}]*)\}/&format_if($1, $node_type)/eg;
    $old_format =~ s/\{Or\}\{([^\}]*)\}/&format_or($1)/eg;
    # $old_format =~ s/\{Or\}\{([^,]*),[^\}]*\}/$1/g;
    $old_format =~ s/\[Text\]/\<gsf:text\/\>/g;
    $old_format =~ s/\[num\]/\<gsf:num\/\>/g;
    $old_format =~ s/\[link\]/\<gsf:link\>/g;
    $old_format =~ s/\[\/link\]/\<\/gsf:link\>/g;
    $old_format =~ s/\[srclink\]/\<gsf:link type=\'source\'\>/g;
    $old_format =~ s/\[\/srclink\]/\<\/gsf:link\>/g;
    $old_format =~ s/\[icon\]/\<gsf:icon\/\>/g;
    $old_format =~ s/\[srcicon\]/\<gsf:icon type=\'source\'\/\>/g;
          
    # what to do with hightlight??
    $old_format =~ s/\[\/?highlight\]//g;

    #now do the rest of the [] which are assumed to be metadata
    $old_format =~ s/\[([^\]]*)\]/&format_metadata($1)/eg;
 #   $old_format =~ s/\[([^\]]*\:)?([^\]\:]*)\]/\<gsf:metadata name=\'$2\' select=\'$1\'\/\>/g;
 #   # remove ex.
 #   $old_format =~ s/<gsf:metadata name=\'ex\.([^\']+)\'/<gsf:metadata name=\'$1\'/g;
 #   #do the parent stuff
 #   $old_format =~ s/(select=\'parent)\:\'/$1\'/g;
 #   $old_format =~ s/select=\'parent\(Top\)\:\'/select=\'root\'/g;
 #   $old_format =~ s/select=\'parent\(All\)\:\'/select=\'ancestors\'/g;
 #   $old_format =~ s/select=\'parent\(All\'([^\']*)\'\)\:\'/select=\'ancestors\' separator=\'$1\'/g;
 #   #remove any select=''
 #   $old_format =~ s/select=\'\'//g;
  
    # some html tidy
    #turn <br> into <br />
    $old_format =~ s/\<br\>/\<br \/\>/g;
    #turn <p> into <p />
    $old_format =~ s/\<p\>/\<p \/\>/g;
    
    #put quotes around any atts
    $old_format =~ s/=([a-z]+)([> ])/=\'$1\'$2/g;

    if ($node_type eq "document") {
    $writer->startTag('gsf:template', 'match'=>'documentNode');
    $writer->charactersXML($old_format);
    $writer->endTag('gsf:template');
    } elsif ($node_type eq "classifier") {
    $writer->startTag('gsf:template', 'match'=>'classifierNode');
    $writer->charactersXML($old_format);
    $writer->endTag('gsf:template');
    } elsif ($node_type eq "horizontal") {  
    $writer->startTag('gsf:template', 'match'=>'classifierNode', 'mode'=>'horizontal');
    $writer->charactersXML($old_format);
    $writer->endTag('gsf:template');

    }
}

sub format_metadata {
    my ($metadata_string) = @_;

    #print STDERR "original meta = $metadata_string\n"; 
    
    # what shall we do with cgisafe??
    my $cgisafe = $metadata_string =~ s/^cgisafe://;

    my ($select) = $metadata_string =~ /^(parent|sibling)/;
    $metadata_string =~ s/^(parent|sibling)//;
    my ($scope, $delim);
    
    if ($select) {
    ($scope) = $metadata_string =~ /^\((Top|All)/;
    $metadata_string =~ s/^\((Top|All)\)?//;
    if ($scope) {
        ($delim) = $metadata_string =~ /^:\'([^\']*)\'\)/;
        $metadata_string =~ s/^:\'([^\']*)\'\)//;
    } 
    }
    $metadata_string =~ s/^://;
    # remove ex.
    $metadata_string =~ s/^ex\.//;
    
    #print STDERR "select=$select, scope=$scope, delim=|$delim|, meta = $metadata_string\n";
    
    my $new_format = "<gsf:metadata name='$metadata_string' ";
    if (defined $select) {
    if ($select eq "sibling") {
        $new_format .= "multiple='true' ";
        if (defined $delim) {
        $new_format .= "separator='$delim' ";
        }
    } elsif ($select eq "parent"){
        if (defined $scope) {
        if ($scope eq "Top") {
            $new_format .= "select='root' ";
        } elsif ($scope eq "All") {
            $new_format .= "select='ancestors' ";
            if (defined $delim) {
            $new_format .= "separator='$delim' ";
            }
        }
        } else {
        $new_format .= "select='parent' ";
        }   
    }
    }
    $new_format .= "/>";
    #print STDERR "$new_format\n";
    return $new_format;
    
}

sub format_if {

    my ($if_string, $node_type) = @_;

    return "if statement to go here";
}

sub format_or {
    my ($or_string) = @_;
    my @meta_list = split (',', $or_string);
    return "" unless scalar (@meta_list);
    my $new_format = "<gsf:choose-metadata>";
    foreach my $m (@meta_list) {
    if ($m =~ /^\[(.*)\]$/) {
        $new_format .= &format_metadata($1);
    } else {
        # a default value
        $new_format .= "<gsf:default>$m</gsf:default>";
        last;
    }
    }
    $new_format .= "</gsf:choose-metadata>";
    return $new_format;
}

sub open_database {
    my ($db_file) = @_;
    
    my $database = ();
    tie (%$database, 'GDBM_File', $db_file, &GDBM_READER, 0400) ||
    die "Couldn't open database $db_file\n";

    return $database;
}

sub close_database {
    my ($database) = @_;
    untie %$database;
}
sub isHorizontalClassifier {
    my ($database, $name) = @_;

    my $record = $database->{$name};
    my ($childtype) = $record =~ /<childtype>(\w*)/;
    if ($childtype eq "HList") {
    return 1;
    }
    return 0;
}
#$writer->startTag('');
#$writer->endTag('');
#$writer->characters();
#$writer->emptyTag('');

1;