Changeset 36670 for main/trunk/greenstone2/perllib/classify/DateList.pm

Timestamp:

2022-10-03T11:14:38+13:00 (19 months ago)

Author:

kjdon

Message:

updated DateList with the changes added for Heritage nz. allow documents with invalid or missing dates to be included in the classifier - if the option is set - in a 'invalid date' partition.

File:

• main/trunk/greenstone2/perllib/classify/DateList.pm (modified) (7 diffs)

main/trunk/greenstone2/perllib/classify/DateList.pm

@@ -34 +34 @@
 # -metadata, use a different metadata for the date (instead of Date), still expects yyyymmdd format. this affects display cos greenstone displays Date metadata as dd month yyyy, whereas any other date metadata is displayed as yyyymmdd - this needs fixing
 # -sort specifies an additional metadata to use in sorting, will take affect when two docs have the same date.
-
+# -reverse_sort - sort in reverse order
+# -no_special_formatting - makes the list a VList instead of a DateList - don't display Months down the side of the list
+# -valid_date_regex - what constitutes a valid date? deft is \d\d\d\d. eg for Heritage, want to customise this to allow dates like 198?
+# -allow_invalid_dates - do we include in the classifier documents with invalid dates? deft = no.
+# -invalid_date_partition_name - if docs with invalid dates are included, they get put into one bucket. this gives the name of that bucket. eg "no date"
 package DateList;
 
@@ -72 +76 @@
     'desc' => "{DateList.no_special_formatting}",
     'type' => "flag",
-    'reqd' => "no" }
-      
+        'reqd' => "no" },
+      { 'name' => "valid_date_regex",
+    'desc' => "{DateList.valid_date_regex}",
+        'type' => "regexp",
+        'deft' => "\\d\\d\\d\\d",
+        'reqd' => "no" },      
+      { 'name' => "allow_invalid_dates",
+    'desc' => "{DateList.allow_invalid_dates}",
+    'type' => "flag",
+        'reqd' => "no" },
+       { 'name' => "invalid_date_partition_name",
+         'desc' => "{DateList.invalid_date_partition_name}",
+         'type' => "string",
+         'deft' => "No Date" }      
       ];
 
@@ -137 +153 @@
     # find the first available metadata
     my $date;
+    my $invalid = 0;
+    my $validre = $self->{'valid_date_regex'};
+
     foreach my $m (@{$self->{'meta_list'}}) {
     $date = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $m);
     last if defined $date;
-    } 
-    
+    }
+
     if (!defined $date || $date eq "") {
-    # if this document doesn't contain Date element we won't 
-    # include it in this classification
-    print $outhandle "DateList: $doc_OID has no date, not including it\n" if $verbosity >=2;
-    return;
+        if (!$self->{'allow_invalid_dates'}) {
+            # if this document doesn't contain Date element we won't 
+            # include it in this classification
+            print $outhandle "DateList: $doc_OID has no date, not including it\n" if $verbosity >=2;
+            return;
+        } else {
+            $invalid = 1;
+            $date = "INVALID";
+        }
     }
 
     # sanity check date
-    if ($date !~ /^\d\d\d\d.*/) {
-    print $outhandle "DateList: $doc_OID date: '$date' malformed: expected it to start with yyyy; not classifying\n" if $verbosity >=2;
-    return;
-    }
-    if ($self->{'bymonth'}) {
+    #if ($date !~ /^\d\d\d\d.*/) {
+    if ($date !~ /^$validre.*/) {
+        if (!$self->{'allow_invalid_dates'}) {
+            print $outhandle "DateList: $doc_OID date: '$date' malformed: expected it to start with $validre; not classifying\n" if $verbosity >=2;
+            return;
+        }
+        else {
+            $invalid = 1;
+            $date = "INVALID";
+        }
+    }
+    if ($self->{'bymonth'} && !$invalid) {
     # check that we have valid month - if not, set it to 00 == undefined
-    if ($date !~ /^\d\d\d\d-?\d\d/) {
+    if ($date !~ /^$validre-?\d\d/) {
     print $outhandle "DateList $doc_OID date: '$date' has no month (expecting yyyymm... or yyyy-mm...), setting date to yyyy-00\n" if $verbosity >=2;
-    $date =~ s/^(\d\d\d\d).*$/$1-00/;
+    $date =~ s/^($validre).*$/$1-00/;
     } else {
-        my ($year, $month) = $date =~ /^(\d\d\d\d)-?(\d\d)/;
+        my ($year, $month) = $date =~ /^($validre)-?(\d\d)/;
         if ($month > 12) {
         print $outhandle "DateList $doc_OID date: '$date' has invalid month, setting date to $year-00\n" if $verbosity >=2;
@@ -236 +267 @@
     }
 
-    
+    my $validre = $self->{'valid_date_regex'};
+    my $invalid_bucket = $self->{'invalid_date_partition_name'};
 
     if ($self->{'bymonth'}) {
@@ -299 +331 @@
     foreach my $classification (@$classlistref) {
         my $date = $self->{'list'}->{$classification};
-        $date =~ s/^(\d\d\d\d).*$/$1/;
+            if ($date =~ /^INVALID/) {
+                $date = $invalid_bucket;
+            } else {
+                $date =~ s/^($validre).*$/$1/;
+            }
         $classhash->{$date} = [] unless defined $classhash->{$date};
         push (@{$classhash->{$date}}, $classification);
@@ -305 +341 @@
           
     }
-    
+
     # only compact the list if nogroup not specified
     if (!$self->{'nogroup'}) {
+        #print STDERR "compacting list\n";
     $classhash = $self->compactlist ($classhash);
     }
@@ -346 +383 @@
     } 
     foreach my $subsection (@subsectionlist) {
+        if ($subsection eq  $self->{'invalid_date_partition_name'}) {
+            # leave this one as is
+        $compactedhash->{$subsection} = $classhashref->{$subsection};
+            next;
+        }
+  #      print STDERR "in sub $subsection\n";
     $currentfirstdate = $subsection if $currentfirstdate eq "";
     if ((scalar (@currentOIDs) < $min) ||