root/main/trunk/greenstone2/perllib/arcinfo.pm @ 32614

Revision 32614, 13.0 KB (checked in by kjdon, 9 months ago)

some changes to arcinfo to handle it holding the reverse lookup db too (from archiveinf-src) - make changes in this object, then write out the complete db at the end of import

  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
Line 
1###########################################################################
2#
3# arcinfo.pm --
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26
27# This module stores information about the archives. At the moment
28# this information just consists of the file name (relative to the
29# directory the archives information file is in) and its OID.
30
31# This module assumes there is a one to one correspondance between
32# a file in the archives directory and an OID.
33
34package arcinfo;
35
36use constant ORDER_OID_INDEX  => 0;
37use constant ORDER_SORT_INDEX => 1;
38
39use constant INFO_FILE_INDEX    => 0;
40use constant INFO_STATUS_INDEX  => 1;
41
42use constant INFO_GROUPPOS_INDEX  => 3;
43use strict;
44
45use dbutil;
46
47
48# File format read in: OID <tab> Filename <tab> Optional-Index-Status
49
50# Index status can be:
51#  I = Index for the first time
52#  R = Reindex
53#  D = Delete
54#  B = Been indexed
55
56sub new {
57    my $class = shift(@_);
58    my $infodbtype = shift(@_);
59
60    # If the infodbtype wasn't passed in, use the default from dbutil
61    if (!defined($infodbtype))
62    {
63      $infodbtype = &dbutil::get_default_infodb_type();
64    }
65
66    my $self = {'infodbtype' => $infodbtype,
67        'info'=>{},
68        'reverse-info'=>{},
69        'order'=>[],
70        'reverse_sort'=>0,
71        'sort'=>0};
72
73    return bless $self, $class;
74}
75
76sub _load_info_txt
77{
78    my $self = shift (@_);
79    my ($filename) = @_;
80
81    if (defined $filename && &FileUtils::fileExists($filename)) {
82    open (INFILE, $filename) ||
83        die "arcinfo::load_info couldn't read $filename\n";
84
85    my ($line, @line);
86    while (defined ($line = <INFILE>)) {
87        $line =~ s/\cM|\cJ//g; # remove end-of-line characters
88        @line = split ("\t", $line); # filename,
89        if (scalar(@line) >= 2) {
90        $self->add_info (@line);
91        }
92    }
93    close (INFILE);
94    }
95
96
97}
98
99sub _load_info_db
100{
101    my $self = shift (@_);
102    my ($filename) = @_;
103
104    my $infodb_map = {};
105
106    &dbutil::read_infodb_file($self->{'infodbtype'}, $filename, $infodb_map);
107
108    foreach my $oid ( keys %$infodb_map ) {
109    my $vals = $infodb_map->{$oid};
110    # interested in doc-file and index-status
111
112    my ($doc_file) = ($vals=~/^<doc-file>(.*)$/m);
113    my ($index_status) = ($vals=~/^<index-status>(.*)$/m);
114    my ($sortmeta) = ($vals=~/^<sort-meta>(.*)$/m);
115    my ($group_position) = ($vals=~/^<group-position>(.*)$/m);
116    $self->add_info ($oid,$doc_file,$index_status,$sortmeta, $group_position);
117    }
118}
119
120
121sub load_info {
122    my $self = shift (@_);
123    my ($filename) = @_;
124
125    $self->{'info'} = {};
126
127    if ((defined $filename) && &FileUtils::fileExists($filename)) {
128    if ($filename =~ m/\.inf$/) {
129        $self->_load_info_txt($filename);
130    }
131    else {
132        $self->_load_info_db($filename);
133    }
134    }
135}
136
137sub _load_filelist_db
138{
139    my $self = shift (@_);
140    my ($filename) = @_;
141
142    my $infodb_map = {};
143
144    &dbutil::read_infodb_file($self->{'infodbtype'}, $filename, $infodb_map);
145
146    foreach my $file ( keys %$infodb_map ) {
147    # turn placeholders in the file keys of archiveinf-src file back to absolute paths
148    $file = &util::placeholders_to_abspath($file);
149    $self->{'prev_import_filelist'}->{$file} = 1;
150    }
151}
152
153
154sub load_prev_import_filelist {
155    my $self = shift (@_);
156    my ($filename) = @_;
157
158    $self->{'import-filelist'} = {};
159
160    if ((defined $filename) && &FileUtils::fileExists($filename)) {
161    if ($filename =~ m/\.inf$/) {
162        # e.g. 'archives-src.inf' (which includes complete list of file
163        # from last time import.pl was run)
164        $self->_load_info_txt($filename);
165    }
166    else {
167        $self->_load_filelist_db($filename);
168    }
169    }
170}
171
172sub load_rev_info
173{
174    my $self = shift (@_);
175    my ($rev_filename) = @_;
176
177    my $rev_infodb_map = {};
178
179    &dbutil::read_infodb_file($self->{'infodbtype'}, $rev_filename, $rev_infodb_map);
180
181    foreach my $srcfile ( keys %$rev_infodb_map ) {
182
183    my $vals = $rev_infodb_map->{$srcfile};
184
185    $srcfile = &util::placeholders_to_abspath($srcfile);
186    foreach my $OID ($vals =~ m/^<oid>(.*)$/gm) {
187        $self->add_reverseinfo($srcfile,$OID);
188    }
189    }
190}
191
192
193sub _save_info_txt {
194    my $self = shift (@_);
195    my ($filename) = @_;
196
197    my ($OID, $info);
198
199    open (OUTFILE, ">$filename") ||
200    die "arcinfo::save_info couldn't write $filename\n";
201 
202    foreach $info (@{$self->get_OID_list()}) {
203    if (defined $info) {
204        print OUTFILE join("\t", @$info), "\n";
205    }
206    }
207    close (OUTFILE);
208}
209
210sub _save_info_db {
211    my $self = shift (@_);
212    my ($filename) = @_;
213
214    my $infodbtype = $self->{'infodbtype'};
215
216    # Not the most efficient operation, but will do for now
217
218    # read it in
219    my $infodb_map = {};
220    &dbutil::read_infodb_file($infodbtype, $filename, $infodb_map);
221
222    # change index-status values
223    foreach my $info (@{$self->get_OID_list()}) {
224    if (defined $info) {
225        my ($oid,$doc_file,$index_status) = @$info;
226        if (defined $infodb_map->{$oid}) {
227        my $vals_ref = \$infodb_map->{$oid};
228        $$vals_ref =~ s/^<index-status>(.*)$/<index-status>$index_status/m;
229        }
230        else {
231        print STDERR "Warning: $filename does not have key $oid\n";
232        }
233    }
234    }
235
236
237    # write out again
238    my $infodb_handle = &dbutil::open_infodb_write_handle($infodbtype, $filename);
239    foreach my $oid ( keys %$infodb_map ) {
240    my $vals = $infodb_map->{$oid};
241    &dbutil::write_infodb_rawentry($infodbtype,$infodb_handle,$oid,$vals);
242    }
243    &dbutil::close_infodb_write_handle($infodbtype, $infodb_handle);
244
245}
246
247sub save_revinfo_db {
248    my $self = shift (@_);
249    my ($rev_filename) = @_;
250    # Output reverse lookup database
251
252    my $rev_infodb_map = $self->{'reverse-info'};
253    my $rev_infodb_handle
254    = &dbutil::open_infodb_write_handle($self->{'infodbtype'}, $rev_filename);
255#   = &dbutil::open_infodb_write_handle($self->{'infodbtype'}, $rev_filename, "append");
256
257    foreach my $key ( keys %$rev_infodb_map ) {
258    my $val_hash = $rev_infodb_map->{$key};
259    $key = &util::abspath_to_placeholders($key);   
260
261    &dbutil::write_infodb_entry($self->{'infodbtype'}, $rev_infodb_handle, $key, $val_hash);
262    }
263    &dbutil::close_infodb_write_handle($self->{'infodbtype'}, $rev_infodb_handle);
264
265}
266
267sub save_info {
268    my $self = shift (@_);
269    my ($filename) = @_;
270
271    if ($filename =~ m/(contents)|(\.inf)$/) {
272    $self->_save_info_txt($filename);
273    }
274    else {
275    $self->_save_info_db($filename);
276    }
277}
278
279sub delete_info {
280    my $self = shift (@_);
281    my ($OID) = @_;
282
283    if (defined $self->{'info'}->{$OID}) {
284    delete $self->{'info'}->{$OID};
285   
286    my $i = 0;
287    while ($i < scalar (@{$self->{'order'}})) {
288        if ($self->{'order'}->[$i]->[ORDER_OID_INDEX] eq $OID) {
289        splice (@{$self->{'order'}}, $i, 1);
290        last;
291        }
292       
293        $i ++;
294    }
295    }
296}
297
298sub add_info {
299    my $self = shift (@_);
300    my ($OID, $doc_file, $index_status, $sortmeta, $group_position) = @_;
301    $sortmeta = "" unless defined $sortmeta;
302    $index_status = "I" unless defined $index_status; # I = needs indexing
303    if (! defined($OID)) {
304    # only happens when no files can be processed?
305    return undef;
306    }
307
308    if (defined $self->{'info'}->{$OID}) {
309    # test to see if we are in a reindex situation
310
311    my $existing_status_info = $self->get_status_info($OID);
312
313    if ($existing_status_info eq "D") {
314        # yes, we're in a reindexing situation
315        $self->delete_info ($OID);
316
317
318        # force setting to "reindex"
319        $index_status = "R";
320
321    }
322    else {
323        # some other, possibly erroneous, situation has arisen
324        # where the document already seems to exist
325        print STDERR "Warning: $OID already exists with index status $existing_status_info\n";
326        print STDERR "         Deleting previous version\n";
327
328        $self->delete_info ($OID);
329    }
330    }
331
332    $self->{'info'}->{$OID} = [$doc_file,$index_status,$sortmeta, $group_position];
333    push (@{$self->{'order'}}, [$OID, $sortmeta]); # ORDER_OID_INDEX and ORDER_SORT_INDEX
334
335
336}
337
338sub set_status_info {
339    my $self = shift (@_);
340    my ($OID, $index_status) = @_;
341
342    my $OID_info = $self->{'info'}->{$OID};
343    $OID_info->[INFO_STATUS_INDEX] = $index_status;
344}
345
346
347sub get_status_info {
348    my $self = shift (@_);
349    my ($OID) = @_;
350
351    my $index_status = undef;
352
353    my $OID_info = $self->{'info'}->{$OID};
354    if (defined $OID_info) {
355    $index_status = $OID_info->[INFO_STATUS_INDEX];
356    }
357    else {
358    die "Unable to find document id $OID\n";
359    }
360
361    return $index_status;
362
363}
364
365sub get_group_position {
366    my $self = shift (@_);
367    my ($OID) = @_;
368
369    my $group_position = undef;
370    my $OID_info = $self->{'info'}->{$OID};
371    if (defined $OID_info) {
372    $group_position = $OID_info->[INFO_GROUPPOS_INDEX];
373    }
374    else {
375    die "Unable to find document id $OID\n";
376    }
377    return $group_position;
378   
379}
380sub add_reverseinfo {
381    my $self = shift (@_);
382    my ($key, $OID) = @_;
383
384    my $existing_key = $self->{'reverse-info'}->{$key};
385    if (!defined $existing_key) {
386    $existing_key = {};
387    $self->{'reverse-info'}->{$key} = $existing_key;
388    }
389
390    my $existing_oid = $existing_key->{'oid'}; 
391    if (!defined $existing_oid) {
392    $existing_oid = [];
393    $existing_key->{'oid'} = $existing_oid;
394    }
395
396    for (@$existing_oid) {
397    if ($_ eq $OID) {
398        return; # already in the list
399    }
400    }
401    push(@$existing_oid,$OID);
402   
403}
404
405sub remove_reverseinfo {
406     my $self = shift (@_);
407     my ($key, $OID) = @_;
408
409     my $existing_key = $self->{'reverse-info'}->{$key};
410     if (!defined $existing_key) {
411     ###print STDERR "trying to remove entry for $key, but its not there!\n";
412     return;
413     }
414     if (!defined $OID) {
415     ###print STDERR "no oid defined, removing whole entry\n";
416     delete $self->{'reverse-info'}->{$key};
417     return;
418     }
419     my $existing_oid = $existing_key->{'oid'};
420     if (!defined $existing_oid) {
421     ###print STDERR "trying to remove entry for $key, but it has no oid field!\n";
422     return;
423     }
424     for my $i (0..scalar(@$existing_oid)) {
425     if (@$existing_oid[$i] eq $OID) {
426         splice @$existing_oid, $i, 1;
427         if (scalar (@$existing_oid) ==0) {
428         ###print STDERRQ "have removed all oids, delete the main key\n";
429         delete $self->{'reverse-info'}->{$key};
430         }
431         return;
432     }
433     }
434}
435sub get_reverseinfo {
436    my $self = shift (@_);
437    my ($key) = @_;
438
439    if ($self->{'reverse-info'}->{$key}) {
440    return $self->{'reverse-info'}->{$key}->{'oid'};
441    }
442    return undef;
443}
444
445sub set_meta_file_flag {
446    my $self = shift (@_);
447    my ($key) = @_;
448
449    my $existing_key = $self->{'reverse-info'}->{$key};
450    if (!defined $existing_key) {
451    $existing_key = {};
452    $self->{'reverse-info'}->{$key} = $existing_key;
453    }
454
455    $existing_key->{'meta-file'} = ["1"];
456
457}
458sub reverse_sort
459{
460    my $self = shift(@_);
461    $self->{'reverse_sort'} = 1;
462}
463sub sort
464{
465    my $self = shift(@_);
466    $self->{'sort'} = 1;
467}
468
469
470# returns a list of the form [[OID, doc_file, index_status], ...]
471sub get_OID_list
472{
473    my $self = shift (@_);
474
475    my $order = $self->{'order'};
476
477    my @sorted_order;
478    if ($self->{'reverse_sort'}) {
479    @sorted_order = sort {$b->[ORDER_SORT_INDEX] cmp $a->[ORDER_SORT_INDEX]} @$order;
480    } elsif ($self->{'sort'}) {
481    @sorted_order = sort {$a->[ORDER_SORT_INDEX] cmp $b->[ORDER_SORT_INDEX]} @$order;
482    } else { # not sorting, don't bother
483    @sorted_order = @$order;
484    }
485
486    my @list = ();
487
488    foreach my $OID_order (@sorted_order) {
489    my $OID = $OID_order->[ORDER_OID_INDEX];
490    my $OID_info = $self->{'info'}->{$OID};
491
492    push (@list, [$OID, $OID_info->[INFO_FILE_INDEX],
493              $OID_info->[INFO_STATUS_INDEX]]);
494    }
495
496    return \@list;
497}
498
499# returns a list of the form [[doc_file, OID], ...]
500sub get_file_list {
501    my $self = shift (@_);
502
503    my $order = $self->{'order'};
504
505    my @sorted_order;
506    if ($self->{'reverse_sort'}) {
507    @sorted_order = sort {$b->[ORDER_SORT_INDEX] cmp $a->[ORDER_SORT_INDEX]} @$order;
508    } elsif ($self->{'sort'}) {
509    @sorted_order = sort {$a->[ORDER_SORT_INDEX] cmp $b->[ORDER_SORT_INDEX]} @$order;
510    } else { # not sorting, don't bother
511    @sorted_order = @$order;
512    }
513
514    my @list = ();
515
516    foreach my $OID_order (@sorted_order) {
517    my $OID = $OID_order->[ORDER_OID_INDEX];
518    my $OID_info = $self->{'info'}->{$OID};
519
520    push (@list, [$OID_info->[INFO_FILE_INDEX], $OID]);
521    }
522
523    return \@list;
524}
525
526
527# returns a list of the form [doc_file,index_status,$sort_meta, $group_position]
528sub get_info {
529    my $self = shift (@_);
530    my ($OID) = @_;
531
532    if (defined $self->{'info'}->{$OID}) {
533    return $self->{'info'}->{$OID};
534    }
535
536    return undef;
537}
538
539
540
541# returns the number of documents so far
542sub size {
543    my $self = shift (@_);
544    return (scalar(@{$self->{'order'}}));
545}
546
5471;
548
Note: See TracBrowser for help on using the browser.