Changeset 10961 for trunk/gsdl/perllib/lucenebuildproc.pm
- Timestamp:
- 2005-11-30T15:29:23+13:00 (18 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/gsdl/perllib/lucenebuildproc.pm
r10474 r10961 60 60 my ($text, $strip_html, $para) = @_; 61 61 62 my ($outtext) = ""; 63 if ($strip_html) { 64 while ($text =~ /<([^>]*)>/ && $text ne "") { 65 66 my $tag = $1; 67 $outtext .= $`." "; #add everything before the matched tag 68 $text = $'; #'everything after the matched tag 69 if ($para && $tag =~ /^\s*p\s/i) { 70 $outtext .= $para; 71 } 72 elsif ($tag =~ /^pre$/) { # a pre tag 73 $text =~ /<\/pre>/; # find the closing pre tag 74 my $tmp_text = $`; #everything before the closing pre tag 75 $text = $'; #'everything after the </pre> 76 $tmp_text =~ s/[<>]//g; # remove all < and > 77 $outtext.= $tmp_text . " "; 78 } 79 } 62 # call the mgpp method first 63 my ($new_text) = $self->SUPER::preprocess_text($text, $strip_html, $para); 64 65 # remove entities 66 $new_text =~ s/&\w{1,10};//g; 67 # remove & 68 $new_text =~ s/&//g; 80 69 81 $outtext .= $text; # add any remaining text 82 } #if strip_html 83 else { 84 $outtext = $text; 85 } 86 #if ($para) { 87 #$text =~ s/(<p\b)/$para$1/gi; 88 #return $text; 89 #} 90 91 # remove entities 92 $outtext =~ s/&\w{1,10};//g; 93 94 return $outtext; 95 } 70 return $new_text; 71 } 72 96 73 97 74 sub text { … … 100 77 my $handle = $self->{'output_handle'}; 101 78 my $outhandle = $self->{'outhandle'}; 102 my $indexed_doc = 1;103 79 104 80 # only output this document if it is one to be indexed 105 81 return if ($doc_obj->get_doc_type() ne "indexed_doc"); 106 82 107 # see if this document belongs to this subcollection 108 foreach my $indexexp (@{$self->{'indexexparr'}}) { 109 $indexed_doc = 0; 110 my ($field, $exp, $options) = split /\//, $indexexp; 111 if (defined ($field) && defined ($exp)) { 112 my ($bool) = $field =~ /^(.)/; 113 $field =~ s/^.// if $bool eq '!'; 114 if ($field =~ /^filename$/i) { 115 $field = $doc_obj->get_source_filename(); 116 } else { 117 $field = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $field); 118 } 119 next unless defined $field; 120 if ($bool eq '!') { 121 if ($options =~ /^i$/i) { 122 if ($field !~ /$exp/i) {$indexed_doc = 1; last;} 123 } else { 124 if ($field !~ /$exp/) {$indexed_doc = 1; last;} 125 } 126 } else { 127 if ($options =~ /^i$/i) { 128 if ($field =~ /$exp/i) {$indexed_doc = 1; last;} 129 } else { 130 if ($field =~ /$exp/) {$indexed_doc = 1; last;} 131 } 132 } 133 } 134 } 135 136 # if this doc is so far in the sub collection, and we have lang info, 137 # now we check the languages to see if it matches 138 if($indexed_doc && defined $self->{'lang_meta'}) { 139 $indexed_doc = 0; 140 my $field = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $self->{'lang_meta'}); 141 if (defined $field) { 142 foreach my $lang (@{$self->{'langarr'}}) { 143 my ($bool) = $lang =~ /^(.)/; 144 if ($bool eq '!') { 145 $lang =~ s/^.//; 146 if ($field !~ /$lang/) { 147 $indexed_doc = 1; last; 148 } 149 } else { 150 if ($field =~ /$lang/) { 151 $indexed_doc = 1; last; 152 } 153 } 154 } 155 } 156 } 83 my $indexed_doc = $self->is_subcollection_doc($doc_obj); 157 84 158 85 # this is another document … … 219 146 # but we need to put the section tag placeholders in there so the 220 147 # sections match up with gdbm db 221 if ($indexed_doc) { 222 #if ($self->{'indexing_text'}) { 223 # $text .= "$parastarttag"; # only add para tags for indexing 224 # note that we assume that metadata will not be asked for for the compressed text, so we add para tags without checking for indexing_text 225 # } 226 $self->{'num_bytes'} += $doc_obj->get_text_length ($section); 227 foreach my $field (split (/,/, $fields)) { 228 # only deal with this field if it doesn't start with top or 229 # this is the first section 230 my $real_field = $field; 231 if (!($real_field =~ s/^top//) || ($doc_section == 1)) { 232 my $new_text = ""; 233 my $tmp_text = ""; 234 if ($real_field eq "text") { 148 if (!$indexed_doc) { 149 $text .= "\n</$sectiontag>\n" if ($sectiontag ne ""); 150 $section = $doc_obj->get_next_section($section); 151 next; 152 } 153 154 $self->{'num_bytes'} += $doc_obj->get_text_length ($section); 155 foreach my $field (split (/;/, $fields)) { 156 # only deal with this field if it doesn't start with top or 157 # this is the first section 158 my $real_field = $field; 159 next if (($real_field =~ s/^top//) && ($doc_section != 1)); 160 161 my $new_text = ""; 162 my $tmp_text = ""; 163 164 # we get allfields by default - do nothing 165 if ($real_field eq "allfields") { 166 167 } 168 # metadata - output all metadata we know about except gsdl stuff 169 elsif ($real_field eq "metadata") { 170 my $shortname = ""; 171 my $metadata = $doc_obj->get_all_metadata ($section); 172 foreach $pair (@$metadata) { 173 my ($mfield, $mvalue) = (@$pair); 174 # check fields here, maybe others dont want - change to use dontindex!! 175 if ($mfield ne "Identifier" 176 && $mfield !~ /^gsdl/ 177 && $mfield ne "classifytype" 178 && $mfield ne "assocfilepath" 179 && defined $mvalue && $mvalue ne "") { 180 181 if (defined $self->{'indexfieldmap'}->{$mfield}) { 182 $shortname = $self->{'indexfieldmap'}->{$mfield}; 183 } 184 else { 185 $shortname = $self->create_shortname($mfield); 186 $self->{'indexfieldmap'}->{$mfield} = $shortname; 187 $self->{'indexfieldmap'}->{$shortname} = 1; 188 } 189 $new_text .= "$parastarttag<$shortname index=\"1\">$mvalue</$shortname>$paraendtag\n"; 190 if (!defined $self->{'indexfields'}->{$mfield}) { 191 $self->{'indexfields'}->{$mfield} = 1; 192 } 193 } 194 } 195 196 } 197 else { 198 #individual metadata and or text specified - could be a comma separated list 199 my $shortname=""; 200 if (defined $self->{'indexfieldmap'}->{$real_field}) { 201 $shortname = $self->{'indexfieldmap'}->{$real_field}; 202 } 203 else { 204 $shortname = $self->create_shortname($real_field); 205 $self->{'indexfieldmap'}->{$real_field} = $shortname; 206 $self->{'indexfieldmap'}->{$shortname} = 1; 207 } 208 209 my @metadata_list = (); 210 foreach $submeta (split /,/, $real_field) { 211 if ($submeta eq "text") { 235 212 if ($self->{'indexing_text'}) { #tag the text with <Text>...</Text>, add the <Paragraph> tags and strip out html if needed 236 $new_text .= "$parastarttag< TXindex=\"1\">\n";213 $new_text .= "$parastarttag<$shortname index=\"1\">\n"; 237 214 $tmp_text .= $doc_obj->get_text ($section); 238 if ($parastarttag =~ "") { 215 if ($parastarttag ne "") { 216 $tmp_text = $self->preprocess_text($tmp_text, $self->{'strip_html'}, "</$shortname>$paraendtag$parastarttag<$shortname index=\"1\">"); 217 } else { 239 218 # we don't want to individually tag each paragraph if not doing para indexing 240 219 $tmp_text = $self->preprocess_text($tmp_text, $self->{'strip_html'}, ""); 241 } else { 242 $tmp_text = $self->preprocess_text($tmp_text, $self->{'strip_html'}, "</TX>$paraendtag$parastarttag<TX index=\"1\">"); 220 } 221 $new_text .= "$tmp_text</$shortname>$paraendtag\n"; 222 } 223 else { # leave html stuff in, but escape the tags, and dont add Paragraph tags - never retrieve paras at the moment 224 $tmp_text .= $doc_obj->get_text ($section); 225 &ghtml::htmlsafe($tmp_text); 226 $new_text .= $tmp_text; 227 228 } 229 } 230 else { 231 my @section_metadata = @{$doc_obj->get_metadata ($section, $submeta)}; 232 if ($self->{'indexing_text'} && defined ($self->{'sections_index_document_metadata'})) { 233 if ($self->{'sections_index_document_metadata'} eq "always" || ( scalar(@section_metadata) == 0 && $self->{'sections_index_document_metadata'} eq "unless_section_metadata_exists")) { 234 push (@section_metadata, @{$doc_obj->get_metadata ($doc_obj->get_top_section(), $submeta)}); 243 235 } 244 245 $new_text .= "$tmp_text</TX>$paraendtag\n"; 246 #if (!defined $self->{'indexfields'}->{'TextOnly'}) { 247 #$self->{'indexfields'}->{'TextOnly'} = 1; 248 #} 249 } 250 else { # leave html stuff in, but escape the tags, and dont add Paragraph tags - never retrieve paras at the moment 251 if ($self->{'store_text'}) { 252 $tmp_text .= $doc_obj->get_text ($section); 253 &ghtml::htmlsafe($tmp_text); 254 $new_text .= $tmp_text; 255 } 256 } 257 } else { # metadata field 258 if ($real_field eq "allfields") { #ignore 259 } 260 elsif ($real_field eq "metadata") { # insert all metadata 261 #except gsdl stuff 262 my $shortname = ""; 263 my $metadata = $doc_obj->get_all_metadata ($section); 264 foreach $pair (@$metadata) { 265 my ($mfield, $mvalue) = (@$pair); 266 # check fields here, maybe others dont want - change to use dontindex!! 267 if ($mfield ne "Identifier" 268 && $mfield !~ /^gsdl/ 269 && $mfield ne "classifytype" 270 && $mfield ne "assocfilepath" 271 && defined $mvalue && $mvalue ne "") { 272 273 if (defined $self->{'indexfieldmap'}->{$mfield}) { 274 $shortname = $self->{'indexfieldmap'}->{$mfield}; 275 } 276 else { 277 $shortname = $self->create_shortname($mfield); 278 $self->{'indexfieldmap'}->{$mfield} = $shortname; 279 $self->{'indexfieldmap'}->{$shortname} = 1; 280 } 281 $new_text .= "$parastarttag<$shortname index=\"1\">$mvalue</$shortname>$paraendtag\n"; 282 if (!defined $self->{'indexfields'}->{$mfield}) { 283 $self->{'indexfields'}->{$mfield} = 1; 284 } 285 } 286 } 287 288 } 289 else { #individual metadata specified 290 my $shortname=""; 291 #if (!defined $self->{'indexfields'}->{$real_field}) { 292 #$self->{'indexfields'}->{$real_field} = 1; 293 #} 294 if (defined $self->{'indexfieldmap'}->{$real_field}) { 295 $shortname = $self->{'indexfieldmap'}->{$real_field}; 296 } 297 else { 298 $shortname = $self->create_shortname($real_field); 299 $self->{'indexfieldmap'}->{$real_field} = $shortname; 300 $self->{'indexfieldmap'}->{$shortname} = 1; 301 } 302 my @section_metadata = @{$doc_obj->get_metadata ($section, $real_field)}; 303 if ($self->{'indexing_text'} && defined ($self->{'sections_index_document_metadata'})) { 304 if ($self->{'sections_index_document_metadata'} eq "always" || ( scalar(@section_metadata) == 0 && $self->{'sections_index_document_metadata'} eq "unless_section_metadata_exists")) { 305 push (@section_metadata, @{$doc_obj->get_metadata ($doc_obj->get_top_section(), $real_field)}); 306 } 307 } 308 foreach $item (@section_metadata) { 309 $new_text .= "$parastarttag<$shortname index=\"1\">$item</$shortname>$paraendtag\n"; 310 # remove entities 311 $new_text =~ s/&\w{1,10};//g; 312 } 313 } 314 236 } 237 push (@metadata_list, @section_metadata); 315 238 } 316 317 # filter the text 318 $self->filter_text ($field, $new_text); 319 $self->{'num_processed_bytes'} += length ($new_text); 320 $text .= "$new_text"; 239 } 240 foreach my $item (@metadata_list) { 241 $new_text .= "$parastarttag<$shortname index=\"1\">$item</$shortname>$paraendtag\n"; 242 # remove entities 243 $new_text =~ s/&\w{1,10};//g; 244 # remove & 245 $new_text =~ s/&//g; 321 246 } 322 247 } 323 } # if (indexed_doc) 324 248 249 # filter the text 250 $self->filter_text ($field, $new_text); 251 $self->{'num_processed_bytes'} += length ($new_text); 252 $text .= "$new_text"; 253 } # foreach field 254 325 255 $text .= "\n</$sectiontag>\n" if ($sectiontag ne ""); 326 256 … … 328 258 } #while defined section 329 259 print $handle "$text\n$documentendtag"; 260 #print STDOUT "$text\n$documentendtag"; 330 261 } 331 262
Note:
See TracChangeset
for help on using the changeset viewer.