Changeset 37187


Ignore:
Timestamp:
2023-01-27T23:52:38+13:00 (15 months ago)
Author:
davidb
Message:

Reworking of file-level document-version history, in light of a clearer understanding of how hardlinking works in terms of inodes on disk. The new solution needs to make use of moving archives to archives_keep, them copying things back. As copying is involved this means time-stamp on the archive infodb used for incremental building can no longer be used to establish which files in 'import' are newer than the last build. The implemented solution here is to store the timestamp of the previous build in a a file (rather than relying on the timestamp of a file created). The opportunity was also taken to record in this file the type of infodb used on that import.pl. With this extra information it is now possible to detect when the type of infodb used has changed in the collectionConfi.xml, meaning import.pl can still function correctly, even in the case of an incremental or incremental-add import.pl being run.

Location:
main/trunk/greenstone2/perllib
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • main/trunk/greenstone2/perllib/DocHistoryFileUtils.pm

    r37149 r37187  
    3838use FileUtils;
    3939
    40 my $fldv_history_dir = "_fldv_history";
     40my $FLDV_HISTORY_DIR = "_fldv_history";
    4141
    4242sub prepend_document_version
    4343{
    4444    my ($keepold_doc_dirname,$doc_dirname) = @_;
    45 
     45   
     46    # Step 5.1:
     47    #   * Deal with the keepold archives doc's in '_fldv_history' first (if it, and the sub-dirs exist)
     48    # Step 5.2:
     49    #   * Deal with the keepold archives doc's top-level content (excluding _fldv_history)
     50
     51    # For $keepold == 1 this means:
     52    #   * Recursive hardlink copy each of the keepold 'nminus-<n> folders -> archives as 'nminus-<n+1>'
     53    #   * Recursive hardlink copy the top-level keepold archive's doc (excluding _fldv_history dir)
     54
     55   
    4656    my $status_ok = 1;
    4757   
    48     my $doc_fldv_history_dirname = &FileUtils::filenameConcatenate($doc_dirname,$fldv_history_dir);
     58    my $doc_fldv_history_dirname = &FileUtils::filenameConcatenate($doc_dirname,$FLDV_HISTORY_DIR);
     59    my $keepold_doc_fldv_history_dirname = &FileUtils::filenameConcatenate($keepold_doc_dirname,$FLDV_HISTORY_DIR);
    4960
    5061    if (&FileUtils::directoryExists($doc_fldv_history_dirname)) {
    51     # need to shuffle nminus-1, nminus-2 down by one
    52     my $matching_dirs = &FileUtils::readDirectoryFiltered($doc_fldv_history_dirname,undef,"^nminus-\\d+\$");
    53    
    54     my @sorted_matching_dirs = sort {
    55         my ($a_num) = ($a =~ m/(\d+)$/);
    56         my ($b_num) = ($b =~ m/(\d+)$/);
    57 
    58         # sort into descending order
    59         return $b_num <=> $a_num;
    60     } @$matching_dirs;
    61 
    62     # want sort order to be higest to lowest, for moving 'n' vals up by one
    63 
    64     foreach my $nminus_n (@sorted_matching_dirs) {
    65 
    66         my $full_dir = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,$nminus_n);
    67         if (-d $full_dir) {
    68         # print STDERR "    Increasing by one the file-level document-version history number for:\n";
    69         # print STDERR "      $full_dir\n";
    70        
    71         my ($n) = ($nminus_n =~ m/(\d+)$/);
    72         my $new_n = $n + 1;
    73 
    74         my $full_new_dir = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,"nminus-$new_n");
    75        
    76         &FileUtils::moveFiles($full_dir,$full_new_dir);
     62    print SDTDERR "Error - The newly formed 'archives' version of document should not contain the existing '$FLDV_HISTORY_DIR' directory:\n";
     63    print STDERR "    $doc_fldv_history_dirname\n";
     64    $status_ok = 0;
     65    }
     66
     67    if ($status_ok) {
     68    # Step 5.1
     69
     70    # Create _fldv_history in archive doc's dir, ready to take the n -> n+1 copies
     71    my $mkdir_ok = &FileUtils::makeDirectory($doc_fldv_history_dirname);
     72
     73    if ($mkdir_ok) {
     74
     75        # If the keepold _fldv_history directory exists, then some prior content to deal with.
     76        # Otherwise, this is our case of first file-level doc-version history for the doc
     77        # and for Step 5.1 there is nothing to do
     78       
     79        if (&FileUtils::directoryExists($keepold_doc_fldv_history_dirname)) {
     80
     81        # Start recursive hardlink copying the directories across as n -> n+1
     82
     83        my $keepold_matching_dirs = &FileUtils::readDirectoryFiltered($keepold_doc_fldv_history_dirname,undef,"^nminus-\\d+\$");
     84
     85        if (scalar(@$keepold_matching_dirs) == 0) {
     86            print STDERR "Warning: no 'nminus-<n>' directories found in $FLDV_HISTORY_DIR:\n";
     87            print STDERR "    $keepold_doc_fldv_history_dirname\n";
     88            print STDERR "=> No file-level document-version history content to increase by 1\n";
     89        }
     90       
     91        my @sorted_keepold_matching_dirs = sort {
     92            my ($a_num) = ($a =~ m/(\d+)$/);
     93            my ($b_num) = ($b =~ m/(\d+)$/);
     94           
     95            # sort into descending order
     96            return $b_num <=> $a_num;
     97        } @$keepold_matching_dirs;
     98       
     99        # want sort order to be higest to lowest, for moving 'n' vals up by one
     100       
     101        foreach my $nminus_n (@sorted_keepold_matching_dirs) {
     102           
     103            my $keepold_full_dir = &FileUtils::filenameConcatenate($keepold_doc_fldv_history_dirname,$nminus_n);
     104            if (-d $keepold_full_dir) {
     105           
     106            my ($n) = ($nminus_n =~ m/(\d+)$/);
     107            my $new_n = $n + 1;
     108           
     109            my $full_new_dir_plus1 = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,"nminus-$new_n");
     110
     111            my $hard_link_ok = &FileUtils::hardlinkFilesRefRecursive([$keepold_full_dir],$full_new_dir_plus1,
     112                                         { 'strict' => 1 } );
     113
     114            if (!$hard_link_ok) {
     115                print STDERR "Error: Failed to recursive hardlink copy 'nminus-<n>' from 'archives_keepold' to 'archives'\n";
     116                $status_ok = 0;
     117                last;
     118            }
     119           
     120            #my $hard_link_ok = &FileUtils::hardlinkFilesRefRecursive([$keepold_full_dir],$full_new_dir_plus1,
     121            #                            { 'strict' => 1, 'exclude_filter_re' => "^$FLDV_HISTORY_DIR\$" } );
     122
     123            #if (!&FileUtils::renameDirectory($full_dir,$full_new_dir)) {
     124            #    print STDERR "Error: Failed to shuffled down 'nminus' dirs\n";
     125            #    $status_ok = 0;
     126            #    last;
     127            #}
     128            }
     129            else {
     130            print STDERR "Warning: skipping $keepold_full_dir as it is not a directory\n";
     131            }
     132        }
     133       
    77134        }
    78135        else {
    79         print STDERR "Warning: skipping $full_dir as it is not a directory\n";
     136        print STDERR "First time file-level document-version history item stored for $doc_fldv_history_dirname\n";
    80137        }
    81138    }
    82    
    83     }
    84     else {
    85     my $mkdir_ok = &FileUtils::makeDirectory($doc_fldv_history_dirname);
    86 
    87     if (!$mkdir_ok) {
    88         print STDERR "Error: DocHistoryFileUtils::prepend_document_version() failed to make directory:\n";
    89         print STDERR "          '$doc_fldv_history_dirname'\n";
    90 
     139    else {
     140        print STDERR "Error: failed to make directory:\n";
     141        print STDERR "    '$doc_fldv_history_dirname'\n";
     142       
    91143        $status_ok = 0;
    92144    }
    93145    }
    94 
     146       
     147
     148   
     149   
     150    #   # Action Step 5 (from inexport.pm)
     151    #   #
     152    #   # 5.1 When $keepold == 1
     153    #   ####   * Move the keepold archives doc's '_fldv_history' to archives (if it exists)
     154    #   ####   * Then, if nminus-1, nminus-2 etc sub-folders exist, shuffle them (in-situ) down by one
     155
     156    #   #   * Recursive hardlink copy each of the keepold 'nminus-<n> folders -> archives as 'nminus-<n+1>'
     157    #   #   * Recursive hardlink copy the top-level keepold archive's doc (excluding _fldv_history dir)
     158   
     159    #   # Move the keepold archives doc's '_fldv_history' to archives (if it exists)
     160    #   #   * Then, if nminus-1, nminus-2 etc sub-folders exist, shuffle them (in-situ) down by one
     161
     162   
     163    #   if (!&FileUtils::directoryExists($keepold_doc_fldv_history_dirname)) {
     164    #       # First time creating a file-level doc-version history folder for this doc
     165    #       # => make new empty _fldv_history folder in archives dir (nothing else to do at this point)
     166    #       my $mkdir_ok = &FileUtils::makeDirectory($doc_fldv_history_dirname);
     167       
     168    #       if (!$mkdir_ok) {
     169    #       print STDERR "Error: failed to make directory:\n";
     170    #       print STDERR "    '$doc_fldv_history_dirname'\n";
     171       
     172    #       $status_ok = 0;
     173    #       }       
     174    #   }
     175    #   else {
     176    #       # Have some existing nminus-n folders to deal this
     177       
     178    #       if (&FileUtils::directoryMove($keepold_$doc_fldv_history_dirname,$doc_fldv_history_dirname)) {
     179
     180    #       # now shuffle nminus-1, nminus-2 down by one
     181    #       my $matching_dirs = &FileUtils::readDirectoryFiltered($doc_fldv_history_dirname,undef,"^nminus-\\d+\$");
     182
     183    #       if (scalar(@$matching_dirs) == 0) {
     184    #           print STDERR "Warning: no 'nminus-<n>' directories found in $FLDV_HISTORY_DIR:\n";
     185    #           print STDERR "    $doc_fldv_history_dirname\n";
     186    #           print STDERR "=> No file-level document-version history to store\n";
     187    #       }
     188       
     189    #       my @sorted_matching_dirs = sort {
     190    #           my ($a_num) = ($a =~ m/(\d+)$/);
     191    #           my ($b_num) = ($b =~ m/(\d+)$/);
     192           
     193    #           # sort into descending order
     194    #           return $b_num <=> $a_num;
     195    #       } @$matching_dirs;
     196       
     197    #       # want sort order to be higest to lowest, for moving 'n' vals up by one
     198       
     199    #       foreach my $nminus_n (@sorted_matching_dirs) {
     200           
     201    #           my $full_dir = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,$nminus_n);
     202    #           if (-d $full_dir) {
     203    #           # print STDERR "    Increasing by one the file-level document-version history number for:\n";
     204    #           # print STDERR "      $full_dir\n";
     205           
     206    #           my ($n) = ($nminus_n =~ m/(\d+)$/);
     207    #           my $new_n = $n + 1;
     208           
     209    #           my $full_new_dir = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,"nminus-$new_n");
     210           
     211    #           if (!&FileUtils::renameDirectory($full_dir,$full_new_dir)) {
     212    #               print STDERR "Error: Failed to shuffled down 'nminus' dirs\n";
     213    #               $status_ok = 0;
     214    #               last;
     215    #           }
     216    #           }
     217    #           else {
     218    #           print STDERR "Warning: skipping $full_dir as it is not a directory\n";
     219    #           }
     220    #       }
     221    #       }
     222    #       else {
     223    #       print STDERR "Error - Failed to move $FLDV_HISTORY_DIR from 'archives_keepold' to 'archives'\n";
     224    #       $status_ok = 0;
     225    #       }
     226    #   }
     227    # }
     228           
    95229    if ($status_ok) {
    96230
     231    # Step 5.2
     232
     233    #   * Recursive hardlink copy the top-level keepold archive's doc (excluding _fldv_history dir)
     234
    97235    my $doc_fldv_history_dirname_nminus1 = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,"nminus-1");
    98236
    99237    my $hard_link_ok = &FileUtils::hardlinkFilesRefRecursive([$keepold_doc_dirname],$doc_fldv_history_dirname_nminus1,
    100                                  { 'strict' => 1, 'exclude_filter_re' => "^$fldv_history_dir\$" } );
    101    
     238                                 { 'strict' => 1, 'exclude_filter_re' => "^$FLDV_HISTORY_DIR\$" } );
     239
    102240    if (!$hard_link_ok) {
    103         print STDERR "Error: DocHistoryFileUtils::prepend_document_version() failed to hardLink\n";
    104         print STDERR "          '$keepold_doc_dirname' -> '$doc_fldv_history_dirname_nminus1'\n";
    105        
     241        print STDERR "Error: Failed to recursively hardlink copy the top-level 'keepold_archives' doc as 'archives' $FLDV_HISTORY_DIR/nminus-1 version\n";
    106242        $status_ok = 0;
    107     }
    108     }
    109 
     243    }   
     244    }
    110245
    111246    if (!$status_ok) {
    112     print STDERR "**** Critical error occurred in creating/updating file-level document-version history\n";
     247    print STDERR "\n";
     248    print STDERR "**** A critical error occurred in creating/updating file-level document-version history\n";
    113249    print STDERR "**** After determining and correcting the cause of the error, to reset, delete\n";
    114     print STDERR "****     your 'archives' folder, and replace it with 'archives_keep'\n";
     250    print STDERR "****     your 'archives' folder, and move 'archives_keep' back to 'archives'\n";
     251    print STDERR "\n";
    115252    exit 1;
    116253    }
    … …  
    120257{
    121258    my ($keepold_doc_dirname,$doc_dirname) = @_;
     259
     260    # Step 5.1:
     261    #   * Deal with the keepold archives doc's in '_fldv_history' first (if it, and the sub-dirs exist)
     262    # Step 5.2:
     263    #   * Deal with the keepold archives doc's top-level content (excluding _fldv_history)
     264   
     265    # For $replaceold == 1 this means:
     266    #   * Starting at 'nminus-2', recursive hardlink copy each of the keepold 'nminus-<n>' folders (keeping 'n' the same value)
     267    #   * Recursive hardlink copy the top-level keepold archive's doc (excluding _fldv_history dir) as 'nminus-1'
    122268   
    123269    my $status_ok = 1;
    124270   
    125     my $doc_fldv_history_dirname = &FileUtils::filenameConcatenate($doc_dirname,$fldv_history_dir);
    126     my $doc_fldv_history_dirname_nminus1 = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,"nminus-1");
    127    
    128     if (!&FileUtils::directoryExists($doc_fldv_history_dirname)) {
     271    my $doc_fldv_history_dirname = &FileUtils::filenameConcatenate($doc_dirname,$FLDV_HISTORY_DIR);
     272    my $keepold_doc_fldv_history_dirname = &FileUtils::filenameConcatenate($keepold_doc_dirname,$FLDV_HISTORY_DIR);
     273   
     274    if (&FileUtils::directoryExists($doc_fldv_history_dirname)) {
     275    print SDTDERR "Error - The newly formed 'archives' version of document should not contain the existing '$FLDV_HISTORY_DIR' directory:\n";
     276    print STDERR "    $doc_fldv_history_dirname\n";
     277    $status_ok = 0;
     278    }
     279
     280
     281    if ($status_ok) {
     282    # Step 5.1
     283
     284    # Create _fldv_history in archive doc's dir, ready to take the n -> n copies, starting at n+2 onwards
    129285    my $mkdir_ok = &FileUtils::makeDirectory($doc_fldv_history_dirname);
    130286
    131     if (!$mkdir_ok) {
    132         print STDERR "Error: DocHistoryFileUtils::replace_document_version() failed to make directory:\n";
    133         print STDERR "          '$doc_fldv_history_dirname'\n";
    134 
     287    if ($mkdir_ok) {
     288
     289        # If the keepold _fldv_history directory exists, then some prior content to deal with.
     290        # As long as there is a nminus-2, then there are sub-folders to copy
     291        # Otherwise there is nothing to do for Step 5.1
     292       
     293        if (&FileUtils::directoryExists($keepold_doc_fldv_history_dirname)) {
     294
     295        # Start recursive hardlink copying the directories across n+2 -> n+3 and onwardds
     296
     297        my $keepold_matching_dirs = &FileUtils::readDirectoryFiltered($keepold_doc_fldv_history_dirname,undef,"^nminus-\\d+\$");
     298
     299        if (scalar(@$keepold_matching_dirs) == 0) {
     300            print STDERR "Warning: no 'nminus-<n>' directories found in $FLDV_HISTORY_DIR:\n";
     301            print STDERR "    $keepold_doc_fldv_history_dirname\n";
     302            print STDERR "=> No file-level document-version history content to increase by 1\n";
     303        }
     304       
     305        my @sorted_keepold_matching_dirs = sort {
     306            my ($a_num) = ($a =~ m/(\d+)$/);
     307            my ($b_num) = ($b =~ m/(\d+)$/);
     308           
     309            # sort into descending order
     310            return $b_num <=> $a_num;
     311        } @$keepold_matching_dirs;
     312       
     313        # want sort order to be higest to lowest, for moving 'n' vals up by one
     314       
     315        foreach my $nminus_n (@sorted_keepold_matching_dirs) {
     316
     317            last if $nminus_n eq "nminus-1";
     318           
     319            my $keepold_full_dir = &FileUtils::filenameConcatenate($keepold_doc_fldv_history_dirname,$nminus_n);
     320            if (-d $keepold_full_dir) {
     321           
     322            # my ($n) = ($nminus_n =~ m/(\d+)$/);
     323           
     324            my $full_new_dir = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,$nminus_n);
     325
     326            my $hard_link_ok = &FileUtils::hardlinkFilesRefRecursive([$keepold_full_dir],$full_new_dir,
     327                                         { 'strict' => 1 } );
     328
     329            if (!$hard_link_ok) {
     330                print STDERR "Error: Failed to recursive hardlink copy 'nminus-<n>' from 'archives_keepold' to 'archives'\n";
     331                $status_ok = 0;
     332                last;
     333            }
     334            }
     335            else {
     336            print STDERR "Warning: skipping $keepold_full_dir as it is not a directory\n";
     337            }
     338        }
     339
     340        }
     341    }
     342    else {
     343        print STDERR "Error: failed to make directory:\n";
     344        print STDERR "    '$doc_fldv_history_dirname'\n";
     345       
    135346        $status_ok = 0;
    136347    }
    137348    }
    138     else {
    139     # Better to upgrade this method to return a ok_status value
    140     &FileUtils::removeFilesRecursive($doc_fldv_history_dirname_nminus1);
    141     }   
    142    
     349
     350    my $doc_fldv_history_dirname_nminus1 = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,"nminus-1");
     351
     352
    143353    if ($status_ok) {
    144354
     355    # Step 5.2
     356
     357    #   * Recursive hardlink copy the top-level keepold archive's doc (excluding _fldv_history dir) to archives as nminus-1
     358
     359    my $doc_fldv_history_dirname_nminus1 = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,"nminus-1");
     360
    145361    my $hard_link_ok = &FileUtils::hardlinkFilesRefRecursive([$keepold_doc_dirname],$doc_fldv_history_dirname_nminus1,
    146                                  { 'strict' => 1, 'exclude_filter_re' => "^$fldv_history_dir\$" } );   
     362                                 { 'strict' => 1, 'exclude_filter_re' => "^$FLDV_HISTORY_DIR\$" } );
     363
    147364    if (!$hard_link_ok) {
    148         print STDERR "Error: DocHistoryFileUtils::replace_document_version() failed to hardLink\n";
    149         print STDERR "          '$keepold_doc_dirname' -> '$doc_fldv_history_dirname_nminus1'\n";
    150        
     365        print STDERR "Error: Failed to recursively hardlink copy the top-level 'keepold_archives' doc as 'archives' $FLDV_HISTORY_DIR/nminus-1 version\n";
    151366        $status_ok = 0;
    152     }
     367    }   
    153368    }
    154369
    155370    if (!$status_ok) {
    156     print STDERR "**** Critical error occurred in creating/updating file-level document-version history\n";
     371    print STDERR "\n";
     372    print STDERR "**** A critical error occurred in creating/updating file-level document-version history\n";
    157373    print STDERR "**** After determining and correcting the cause of the error, to reset, delete\n";
    158     print STDERR "****     your 'archives' folder, and replace it with 'archives_keep'\n";
     374    print STDERR "****     your 'archives' folder, and move 'archives_keep' back to 'archives'\n";
     375    print STDERR "\n";
    159376    exit 1;
    160     }
     377    }   
    161378}
    162379
    … …  
    167384    my ($collectcfg, $keepold,$replaceold,$incremental_mode, $archive_info,$archivedir, $archivedir_keepold) = @_;
    168385
     386    # Action Step 5 (from inexport.pm)
     387   
     388    # 5.1  a keepold doc's '_fldv_history' goes first
     389    # 5.2  then the keepold doc's top-level content for new 'nminus 1'
     390   
    169391    my $perform_firsttime_init = 1;
    170392    my $arcinfo_keepold_doc_filename = &dbutil::get_infodb_file_path($collectcfg->{'infodbtype'}, "archiveinf-doc", $archivedir_keepold, $perform_firsttime_init);
    … …  
    184406    my $keepold_doc_file     = $keepold_info_array->[1];
    185407    my $keepold_index_status = $keepold_info_array->[2];
    186    
    187    
     408       
    188409    # Work through all entries in arcinfo_keepold doc-id entries:
    189     #
    190     #   If keepold entry does *not* exist in (the more up to date) archive_info
    191     #   => print out an error.  This should not happen!
    192    
    193     #   If keepold entry *does* exist in (the more up to date) archive_info
    194410    #
    195411    #   (1) if archive_info entry is marked for deletion (D)
    … …  
    199415    #
    200416    #   (2) if -keepold on
    201     #      => Use hard-linking to turn 'archives_keepold' doc folder
    202     #         (without its fldv-history folder) into fldv-history/nminus-1 in the
    203     #         the 'archives' doc folder, having first shuffled any existing
    204     #         nminus-1, nminus-2 folders down by one value
     417    #      => move 'archives_keepold' doc's _fldv_history folder to 'archives' version
     418    #         shuffle all the numbers down by one
     419    #         move the 'archives_keepold' doc's folder to be the archives _fldv-history/nminus-1
    205420    #
    206421    #   (3) if -replaceold on
    … …  
    258473            if ($index_status eq "R") {
    259474            print STDERR "  Updated version of document from import directory (Index-Status=R)\n";
    260             print STDERR "  => keepold: storing snapshot of previous version in $fldv_history_dir as 'nminus-1'\n";
     475            print STDERR "  => keepold: storing snapshot of previous version in $FLDV_HISTORY_DIR as 'nminus-1'\n";
    261476            prepend_document_version($keepold_doc_dirname,$doc_dirname);
    262477            }
    263478            elsif ($index_status eq "I") {
    264479            print STDERR "  Newly generated version of document from 'import/' of existing document from 'archives/' (Index-Status=I)\n";
    265             print STDERR "  => keepold: store snapshot of previous version in $fldv_history_dir as 'nminus-1'\n";
     480            print STDERR "  => keepold: store snapshot of previous version in $FLDV_HISTORY_DIR as 'nminus-1'\n";
    266481            prepend_document_version($keepold_doc_dirname,$doc_dirname);
    267482            }
    … …  
    269484            if ($incremental_mode ne "all") {
    270485                print STDERR "  Unchanged version of document in 'archives/' (Index-Status=B)\n";
    271                 print STDERR "  => keepold without incremental: store snapshot of previous version in $fldv_history_dir as 'nminus-1'\n";
     486                print STDERR "  => keepold without incremental: store snapshot of previous version in $FLDV_HISTORY_DIR as 'nminus-1'\n";
    272487                prepend_document_version($keepold_doc_dirname,$doc_dirname);               
    273488            }
    … …  
    290505            if ($index_status eq "R") {
    291506            print STDERR "  Updated version of document from import directory (Index-Status=R)\n";
    292             print STDERR "  => replaceold: replacing snapshot of previous version in $fldv_history_dir as 'nminus-1'\n";
     507            print STDERR "  => replaceold: replacing snapshot of previous version in $FLDV_HISTORY_DIR as 'nminus-1'\n";
    293508            replace_document_version($keepold_doc_dirname,$doc_dirname);
    294509            }
    295510            elsif ($index_status eq "I") {
    296511            print STDERR "  Newly generated version of document from 'import/' of existing document from 'archives/' (Index-Status=I)\n";
    297             print STDERR "  => keepold: store snapshot of previous version in $fldv_history_dir as 'nminus-1'\n";
     512            print STDERR "  => keepold: store snapshot of previous version in $FLDV_HISTORY_DIR as 'nminus-1'\n";
    298513            replace_document_version($keepold_doc_dirname,$doc_dirname);
    299514            }
    … …  
    301516            if ($incremental_mode ne "all") {
    302517                print STDERR "  Unchanged version of document in 'archives/' (Index-Status=B)\n";
    303                 print STDERR "  => keepold without incremental: store snapshot of previous version in $fldv_history_dir as 'nminus-1'\n";
     518                print STDERR "  => keepold without incremental: store snapshot of previous version in $FLDV_HISTORY_DIR as 'nminus-1'\n";
    304519                replace_document_version($keepold_doc_dirname,$doc_dirname);               
    305520            }

  • main/trunk/greenstone2/perllib/FileUtils.pm

    r37151 r37187  
    120120# internal version that copies a file or a group of files
    121121#
    122 sub _copyFilesGeneral
     122sub copyFilesGeneral
    123123{
    124124    my ($srcfiles_ref,$dest,$options) = @_;
    … …  
    137137    if (scalar(@$srcfiles_ref) == 0)
    138138    {
    139     print STDERR "FileUtils::_copyFilesGeneral() no destination directory given\n";
     139    print STDERR "FileUtils::copyFilesGeneral() no destination directory given\n";
    140140    return 0;
    141141    }
    142142    elsif ((scalar(@$srcfiles_ref) > 1) && (!-d $dest))
    143143    {
    144     print STDERR "FileUtils::_copyFilesGeneral() if multiple source files are given the destination must be a directory\n";
     144    print STDERR "FileUtils::copyFilesGeneral() if multiple source files are given the destination must be a directory\n";
    145145    return 0;
    146146    }
    … …  
    159159    if (!-e $file)
    160160    {
    161         print STDERR "FileUtils::_copyFilesGeneral() $file does not exist\n";
     161        print STDERR "FileUtils::copyFilesGeneral() $file does not exist\n";
    162162        $had_an_error = 1;
    163163        if ($strict) {
    … …  
    167167    elsif (!-f $file)
    168168    {
    169         print STDERR "FileUtils::_copyFilesGeneral() $file is not a regular file\n";
     169        print STDERR "FileUtils::copyFilesGeneral() $file is not a regular file\n";
    170170        $had_an_error = 1;
    171171        if ($strict) {
    … …  
    181181        if (!link($file, $tempdest))
    182182        {
    183             print STDERR "Warning: FileUtils::_copyFilesGeneral(): unable to create hard link. ";
     183            print STDERR "Warning: FileUtils::copyFilesGeneral(): unable to create hard link. ";
    184184            print STDERR "  Attempting file copy: $file -> $tempdest\n";
    185185            $success = &File::Copy::copy($file, $tempdest);
    … …  
    195195       
    196196        if (!$success) {
    197         print STDERR "FileUtils::_copyFilesGeneral() failed to copy $file -> $tempdest\n";
     197        print STDERR "FileUtils::copyFilesGeneral() failed to copy $file -> $tempdest\n";
    198198        $had_an_error = 1;
    199199       
    … …  
    226226  my (@srcfiles) = @_;
    227227 
    228   return &_copyFilesGeneral(\@srcfiles,$dest,undef);
     228  return &copyFilesGeneral(\@srcfiles,$dest,undef);
    229229}
    230230
    … …  
    306306    my $strict = 0;
    307307    my $make_fullpath = 0;
     308    my $exclude_dirs  = 0;
     309    my $exclude_files = 0;
    308310    my $exclude_filter_re = undef;
    309311    my $include_filter_re = undef;
    … …  
    312314    $strict = $options->{'strict'} if defined $options->{'strict'};
    313315    $make_fullpath = $options->{'make_fullpath'} if defined $options->{'make_fullpath'};
     316    $exclude_dirs  = $options->{'exclude_dirs'}  if defined $options->{'exclude_dirs'};
     317    $exclude_files = $options->{'exclude_files'} if defined $options->{'exclude_files'};
    314318    $exclude_filter_re = $options->{'exclude_filter_re'} if defined $options->{'exclude_filter_re'};
    315319    $include_filter_re = $options->{'include_filter_re'} if defined $options->{'include_filter_re'};
    … …  
    330334    {
    331335        next if $f_or_d =~ /^\.\.?$/;
     336        next if $exclude_dirs  && -d &filenameConcatenate($src_dir_fullpath, $f_or_d);
     337        next if $exclude_files && -f &filenameConcatenate($src_dir_fullpath, $f_or_d);     
    332338        next if (defined $exclude_filter_re && ($f_or_d =~ m/$exclude_filter_re/));
    333339       
    … …  
    556562            {
    557563                my $fullpath_subf = $fullpath_subf_or_subd;
    558                 my $ret_val_success = &_copyFilesGeneral([$fullpath_subf],$dest,$options);
     564                my $ret_val_success = &copyFilesGeneral([$fullpath_subf],$dest,$options);
    559565
    560566                if ($ret_val_success == 0) {
    … …  
    585591    else
    586592    {
    587         my $ret_val_success = &_copyFilesGeneral([$file], $dest, $options);
     593        my $ret_val_success = &copyFilesGeneral([$file], $dest, $options);
    588594        if ($ret_val_success == 0) {
    589595
    … …  
    14871493## moveFiles()
    14881494
     1495
     1496## @function renameDirectory()
     1497#
     1498# rename a directory
     1499# (effectively a move, where the destination name cannot already exist)
     1500#
     1501sub renameDirectory
     1502{
     1503    my ($srcdir,$dstdir) = @_;
     1504
     1505    my $had_an_error = 0;
     1506
     1507    if (!-d $srcdir) {
     1508    print STDERR "FileUtils::renameDirectory() Error - Source name must be an existing directory\n";
     1509    print STDERR "Source name was: $srcdir\n";
     1510    $had_an_error = 1;
     1511    }
     1512    elsif (-e $dstdir) {
     1513    print STDERR "FileUtils::renameDirectory() Error - Destination name must not already exist\n";
     1514    print STDERR "Destination name was: $dstdir\n";
     1515    $had_an_error = 1;
     1516
     1517    }
     1518    else {
     1519    if (!rename($srcdir,$dstdir)) {
     1520        print STDERR "FileUtils::renameDirectory() -- Error occured moving source name to destination name\n";
     1521        print STDERR "Source name was: $srcdir\n";
     1522        print STDERR "Destination name was: $dstdir\n";
     1523        $had_an_error = 1;
     1524    }
     1525    }
     1526   
     1527    if ($had_an_error) {
     1528    return 0; # i.e., not OK!
     1529    }
     1530    else {
     1531    return 1;
     1532    }
     1533}
     1534## renameDirectory()
     1535 
    14891536## @function openFileHandle()
    14901537#

  • main/trunk/greenstone2/perllib/arcinfo.pm

    r37150 r37187  
    6565
    6666    my $self = {'infodbtype' => $infodbtype,
    67         'info'=>{},
    68         'reverse-info'=>{},
    69         'order'=>[],
    70         'reverse_sort'=>0,
    71         'sort'=>0};
     67        'timestamp_file' => "archiveinf-timestamp.out",
     68        'info'        => {},
     69        'reverse-info'=> {},
     70        'order'       => [],
     71        'reverse_sort'=> 0,
     72        'sort'        => 0};
    7273
    7374    return bless $self, $class;
    … …  
    9394    close (INFILE);
    9495    }
    95 
    96 
    9796}
    9897
    … …  
    123122    my ($filename) = @_;
    124123
     124    my ($cpackage,$cfilename,$cline,$csubr,$chas_args,$cwantarray) = caller(1);
     125    print STDERR "\n\n!!!!!!!!!!!! load_info() -- Calling method: $cfilename:$cline $cpackage->$csubr\n\n";
     126   
    125127    $self->{'info'} = {};
    126128
    127     if ((defined $filename) && &FileUtils::fileExists($filename)) {
    128     if ($filename =~ m/\.inf$/) {
    129         $self->_load_info_txt($filename);
    130     }
    131     else {
    132         $self->_load_info_db($filename);
    133     }
    134     }
     129    if (defined $filename) {
     130   
     131    my $timestamp_filename = $self->get_timestamp_filename($filename);
     132    if (&FileUtils::fileExists($timestamp_filename)) {
     133        my ($prev_infodbtype,$unused_timestamp) = $self->load_timestamp($filename);
     134        $self->{'prev-infodbtype'} = $prev_infodbtype;
     135    }
     136
     137    if (!&FileUtils::fileExists($filename)) {
     138        # Typically a sign of a first-time, or -removeold build
     139
     140        # ... but could be because the value of infodbtype has changed since the last build
     141        # => Check
     142
     143        my $prev_infodbtype = $self->{'prev-infodbtype'};
     144        my $infodbtype = $self->{'infodbtype'};
     145       
     146        if (defined $prev_infodbtype && ($prev_infodbtype ne $infodbtype)) {
     147        print STDERR "arcinfo::load_info() detected change in infodbtype from previous build: $prev_infodbtype -> $infodbtype\n";
     148        print STDERR "=> adjusting archiveinf filename accordingly.\n";
     149
     150        $filename =~ s/\.$infodbtype$/.$prev_infodbtype/;
     151        }
     152    }
     153
     154    # $filename might have changed if infodbtypes don't match
     155    if (&FileUtils::fileExists($filename)) {
     156        if ($filename =~ m/\.inf$/) {
     157        $self->_load_info_txt($filename);
     158        }
     159        else {
     160        $self->_load_info_db($filename);
     161        }
     162    }
     163    }   
    135164}
    136165
    … …  
    274303sub save_info {
    275304    my $self = shift (@_);
    276     my ($filename) = @_;
    277 
    278     if ($filename =~ m/(contents)|(\.inf)$/) {
    279     $self->_save_info_txt($filename);
     305    my ($arcinfo_filename) = @_;
     306
     307   
     308    if ($arcinfo_filename =~ m/(contents)|(\.inf)$/) {
     309    $self->_save_info_txt($arcinfo_filename);
    280310    }
    281311    else {
    282     $self->_save_info_db($filename);
    283     }
    284 }
     312    $self->_save_info_db($arcinfo_filename);
     313    }
     314}
     315
     316
     317
     318sub get_timestamp_filename
     319{
     320    my $self = shift (@_);
     321    my ($arcinfo_doc_filename) = @_;
     322
     323    my $arcinfo_dirname = &File::Basename::dirname($arcinfo_doc_filename);
     324   
     325    my $timestamp_filename = &FileUtils::filenameConcatenate($arcinfo_dirname, $self->{'timestamp_file'});
     326
     327    return $timestamp_filename;
     328}
     329
     330
     331sub load_timestamp
     332{
     333    my $self = shift (@_);
     334    my ($arcinfo_doc_filename) = @_;
     335
     336    my $arcinfo_timestamp_filename = $self->get_timestamp_filename($arcinfo_doc_filename);
     337   
     338    my $timestamp_infodbtype = undef;
     339    my $arcinfo_timestamp    = undef;
     340   
     341    if (-f $arcinfo_timestamp_filename) {
     342    if (open(ARCINFTS_IN,"<$arcinfo_timestamp_filename")) {
     343        my $line = <ARCINFTS_IN>;
     344        chomp $line;
     345        close(ARCINFTS_IN);
     346
     347        ($timestamp_infodbtype,$arcinfo_timestamp) = ($line =~ m/^\s*(\w+)\s*:\s*(\d+)\s*$/)
     348    }
     349    else {
     350        print STDERR "Error - failed to read:\n";
     351        print STDERR "    $arcinfo_timestamp_filename\n";
     352    }
     353    }
     354   
     355    return ($timestamp_infodbtype,$arcinfo_timestamp);
     356}
     357
     358
     359sub save_arcinfo_doc_timestamp
     360{
     361    my $self = shift (@_);
     362    my ($arcinfo_doc_filename) = @_;
     363    print STDERR "\n\n!!!!!!!!!! SAVE save_arcinfo_doc_timestamp()\n\n";
     364   
     365    my $arcinfo_timestamp_filename = $self->get_timestamp_filename($arcinfo_doc_filename);
     366
     367    my $infodbtype = $self->{'infodbtype'};
     368
     369    # Before recording the current infodbtype type and timestamp
     370    # handle the case of prev_infodbtype being different to infodbyte
     371    # => mothball the prev_infodbtype as a backup file
     372   
     373    my $prev_infodbtype = $self->{'prev-infodbtype'};
     374    if ((defined $prev_infodbtype) && ($prev_infodbtype ne $infodbtype)) {
     375    my ($arcinfo_tailname, $arcinfo_dirname, $arcinfo_suffix)
     376        = &File::Basename::fileparse($arcinfo_doc_filename, "\\.[^\\.]+\$");
     377
     378    my $prev_arcinfo_filename   = &dbutil::get_infodb_file_path($prev_infodbtype,$arcinfo_tailname,$arcinfo_dirname);   
     379    my $backup_arcinfo_filename = $prev_arcinfo_filename;
     380    $backup_arcinfo_filename =~ s/\.$prev_infodbtype$/.bak.$prev_infodbtype/;
     381
     382    print STDERR "As a result of infodbtype changing from $prev_infodbtype -> $infodbtype\n";   
     383    print STDERR "=> Making a backup copy of $prev_arcinfo_filename\n";
     384    &dbutil::rename_db_file_to($prev_infodbtype,$prev_arcinfo_filename,$backup_arcinfo_filename);
     385
     386    }
     387
     388    # Could consider taking the the timestamp of the file itself ???
     389    #   e.g. $arcinfo_timestamp = -M $arcinfo_filename;
     390    # But for now base this on a 'live' reading
     391    my $arcinfo_timestamp = time();
     392   
     393    if (open(ARCINFTS_OUT,">$arcinfo_timestamp_filename")) {
     394    print ARCINFTS_OUT "$infodbtype:$arcinfo_timestamp\n";
     395   
     396    close(ARCINFTS_OUT);
     397    }
     398    else {
     399    print STDERR "Error - failed to write:\n";
     400    print STDERR "    $arcinfo_timestamp_filename\n";
     401    }
     402}
     403
    285404
    286405sub delete_info {

  • main/trunk/greenstone2/perllib/inexport.pm

    r37178 r37187  
    521521    my $archivedir  = $self->{'archivedir'} || $self->{'exportdir'};
    522522    # 'archivedir' is a tad abused, and is sometimes set to the 'exportdir' value,
    523     # meaining 'archivedir_keepold' is actually the export dir name with '_keepold' appended
     523    # however at this stage in the code development'archivedir_keepold' is only associated with archivedir (used to provide fldv-history)
    524524    my $archivedir_keepold  = $self->{'archivedir_keepold'};
    525525
    … …  
    655655    else { 
    656656    # If not $removeold, then must be $keepold or $replaceold
    657     # => for either case want to "hard-link"/copy 'archives' to 'archives_keepold'
    658 
    659     # Want to be super careful about doing this, so as not to accidentally
    660     # wipe out any previous file-level document-version history
    661 
     657    # => for either case the sequence to go through is:
     658    #
     659    # 1. Move 'archives' to 'archives_keepold'
     660    # 2. Create new empty 'archives'
     661    # 3. Copy top-level files in 'archives_keepold' to 'archives';
     662    # 4. Allow 'import' to populate 'archives' as usual
     663    #
     664    # 5. Resolve file-level document-verison history through
     665    #    "hard-link"/copy content from 'archives_keep' back to 'archives'
     666    # 5.1  a keepold doc's '_fldv_history' goes first
     667    # 5.2  then the keepold doc's top-level content for new 'nminus 1'
     668   
     669    # Only if all these stages run without a single error then is
     670    # it then safe to remove archivedir_keepold
     671
     672    # If an error occurs, the process is stopped, and deleting
     673    # 'archives' and moving 'archives_keepold' restores things
     674    # back to how they were before import.pl was run.
     675
     676   
    662677    # If got to here, then there is no pre-existing $archivedir_keepold
    663     # => Hard-link copy the contents of 'archives' to 'archives_keepold'
    664     # => Stop if there is any issue with creating the hard-link copy
    665    
    666     if (!&FileUtils::hardlinkFilesRefRecursive([$archivedir],$archivedir_keepold, { 'strict' => 1 } )) {
    667        
     678    # Action Step 1.
     679
     680   
     681    if (!rename($archivedir,$archivedir_keepold)) {
     682
    668683        &gsprintf(STDERR, "\nError message: $!\n\n");
    669684       
    670         &gsprintf(STDERR, "**** Failed to make a hard-link copy of:\n");
     685        &gsprintf(STDERR, "**** Failed to move:\n");
    671686        &gsprintf(STDERR, "****     $archivedir\n");
    672687        &gsprintf(STDERR, "**** to:\n");
    … …  
    679694    }
    680695
    681     # create the archives dir if needed
     696    # Create the archives dir if needed
     697    # coincidentally fldv-history: Action Step 2
    682698    &FileUtils::makeAllDirectories($archivedir);
    683699
    684     # read the archive information file
    685 
     700    if ($keepold || $replaceold) {
     701    # fldv-history: Action Step 3
     702
     703    my ($ret_val_success,$fullpath_files) = &FileUtils::readdirFullpath($archivedir_keepold, { 'strict' => 1, 'exclude_dirs' => 1 });
     704       
     705    my $copy_ok = &FileUtils::copyFilesGeneral($fullpath_files,$archivedir, { 'strict' => 1 });
     706    if (!$copy_ok) {
     707        &gsprintf(STDERR, "**** Failed to copy top-leve files from:\n");
     708        &gsprintf(STDERR, "****     $archivedir_keepold\n");
     709        &gsprintf(STDERR, "**** to:\n");
     710        &gsprintf(STDERR, "****     $archivedir\n");
     711        &gsprintf(STDERR, "****\n");
     712        &gsprintf(STDERR, "**** Unable to proceed with file-level document-version history $inexport_mode => Stopping\n");
     713
     714        exit 1;     
     715    }
     716
     717    print STDERR "\n\n\n";
     718    print STDERR "*****!!!!! No easy way in perl to perform file copy and perserve timestamps\n";
     719    print STDERR "*****!!!!! SO => need to implement file with timestamp within (and DBinfo type for good measure)\n";
     720    print STDERR "*****!!!!!       and change plugin/incremental building that depends on/uses -M \n";
     721    # ArchiveInfoPlugin, DirectoryPlugin inexport.pm, arcinfo.pma
     722    # DirectoryPlugin inexport.pm, arcinfo.pm (convertutil.pm OK, as working on the two files passed to it)
     723    print STDERR "\n\n\n";
     724    }
     725
     726
     727    # Read the archive information file
     728    # coincidentally fldv-history: Action Step 4
     729   
    686730    # BACKWARDS COMPATIBILITY: Just in case there are old .ldb/.bdb files (won't do anything for other infodbtypes)
    687731    &util::rename_ldb_or_bdb_file(&FileUtils::filenameConcatenate($archivedir, "archiveinf-doc"));
    … …  
    11031147        $archive_info->save_revinfo_db($arcinfo_src_filename);
    11041148    }
     1149
     1150    $archive_info->save_arcinfo_doc_timestamp($arcinfo_doc_filename);
    11051151    }
    11061152
    … …  
    11111157
    11121158    if ($keepold || $replaceold) {
     1159
     1160    # fldv-history: Action Step 5
    11131161
    11141162    &DocHistoryFileUtils::archivedir_keepold_to_archivedir($collectcfg, $keepold, $replaceold, $incremental_mode, $archive_info, $archivedir,$archivedir_keepold);
    … …  
    12901338    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archivedir);
    12911339
    1292     my $archiveinf_timestamp = -M $arcinfo_doc_filename;
     1340    #my $archiveinf_timestamp = -M $arcinfo_doc_filename;
     1341    my ($unused_infodbtype,$archiveinf_timestamp) = $archive_info->load_timestamp($arcinfo_doc_filename);
    12931342
    12941343    # First convert all files to absolute form
    … …  
    16091658
    16101659
     1660
     1661
    161116621;

  • main/trunk/greenstone2/perllib/plugins/DirectoryPlugin.pm

    r36372 r37187  
    136136    $infodbtype = "gdbm" if $infodbtype eq "gdbm-txtgz"; # in archives, cannot use txtgz version
    137137    my $output_dir = $processor->getoutputdir();
    138         my $archives_inf = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $output_dir);
    139 
    140     if ( -e $archives_inf ) {
    141         $self->{'inf_timestamp'} = -M $archives_inf;
     138    my $arcinfo_doc_filename   = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $output_dir);
     139
     140    if ( -e $arcinfo_doc_filename ) {
     141        my $archive_info = new arcinfo($infodbtype);
     142        my ($unused_infodbtype,$archiveinf_timestamp) = $archive_info->load_timestamp($arcinfo_doc_filename);
     143       
     144        # $self->{'inf_timestamp'} = -M $archives_inf;
     145        $self->{'inf_timestamp'} = $archiveinf_timestamp;
    142146    }
    143147    }
Note: See TracChangeset for help on using the changeset viewer.