Show
Ignore:
Timestamp:
14.08.2013 22:02:17 (6 years ago)
Author:
ak19
Message:

Major overhaul to script generating the model-collections. This script works, but the version with updated comments and a diff rq at the end will be committed hereafter

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • other-projects/nightly-tasks/diffcol/trunk/gen-model-colls.sh

    r28046 r28048  
    6262# --svnupdate will leave untouched any files and folders unique to model-collect. 
    6363 
    64 # svn update/delete a single collection 
    65 function update_single_collection () { 
     64# See earlier version of this script: 
     65# To svn remove what's unique to model-collect and svn add what's been rebuilt in index and archives 
     66# see http://stackoverflow.com/questions/7502261/delete-folder-content-and-remove-from-version-control 
     67 
     68# http://stackoverflow.com/questions/5044214/how-do-i-detect-and-or-delete-empty-subversion-directories 
     69# http://stackoverflow.com/questions/1301203/removing-svn-files-from-all-directories 
     70 
     71 
     72# DON'T ADD ANY FURTHER ECHO STATEMENTS IN FUNCTION get_col_basename 
     73# "you have to be really careful on what you have in this function, as having any code which will eventually echo will mean that you get incorrect return string." 
     74# see http://stackoverflow.com/questions/3236871/how-to-return-a-string-value-from-a-bash-function 
     75function get_col_basename () { 
    6676    collection=$1 
    67     mode=$2 
    6877 
    6978    #escape the filename (in case of space) 
     
    7382    collection=`basename $collection` 
    7483 
     84    # returning a string does not work in bash 
     85    # see http://stackoverflow.com/questions/3236871/how-to-return-a-string-value-from-a-bash-function     
     86 
     87    #return $collection 
     88    echo $collection 
     89} 
     90 
     91 
     92# Function that handles the --svndelete flag (mode) of this script 
     93function svn_delete () { 
     94 
     95    # svn remove archives and index in each collection 
     96    # commit them all 
     97    # copy over newly rebuilt archives and index into each model-collection 
     98    # svn add the new archives and index folders of each collection 
     99    # commit them all 
     100     
     101     
     102    if [ "x$1" == "x" ]; then 
     103    for collection in collect/*; do      
     104        _del_col_archives_index $collection 
     105    done 
     106    else  
     107    for collection in "$@"; do 
     108        _del_col_archives_index $collection 
     109    done 
     110    fi 
     111 
     112    # commit all the svn rm statements done above in one go: 
     113    # don't do `svn up` here, as this will then retrieve all the folders that were svn-removed 
     114    svn commit -m "Clean rebuild of model collections 1/2. Clearing out deprecated archives and index." model-collect 
     115 
     116    # do an svn up to locally delete what was svn-removed above, BEFORE copying from the rebuilt archives and index folders 
     117    svn up model-collect 
     118 
     119    if [ "x$1" == "x" ]; then 
     120    for collection in collect/*; do      
     121        _add_col_archives_index $collection 
     122    done 
     123    else  
     124    for collection in "$@"; do 
     125        _add_col_archives_index $collection 
     126    done 
     127    fi 
     128 
     129    # commit all the svn add statements done just above in one go 
     130    svn commit -m "Clean rebuild of model collections 2/2. Adding rebuilt archives and index." model-collect 
     131} 
     132 
     133# To undo the changes made by svndelete, run the following manually 
     134# svn revert --depth infinity <model-collect/$collection/archives/* 
     135# svn revert --depth infinity <model-collect/$collection/archives/* 
     136# then remove both the local archives and index, and do an svn up to get original checkout back 
     137 
     138# svn delete this collection's archives and index folders 
     139# (The commit will be done when in one step for all collections on which this function was called) 
     140function _del_col_archives_index () { 
     141    collection=$1 
     142 
     143    #escape the filename (in case of space) and get just the basename 
     144    collection=$(get_col_basename $collection) 
     145 
    75146    if [ ! -e model-collect/$collection ]; then 
    76     echo "$collection does not exist in model-collect, commit separately" 
     147    echo "del_col_archives_index: $collection does not exist in model-collect, will svn add this new collection shortly" 
     148    return; 
     149    fi 
     150 
     151    # remove the entire archives and index folders from svn 
     152    svn rm model-collect/$collection/archives 
     153    svn rm model-collect/$collection/index 
     154 
     155# for TESTING purposes: 
     156#    rm -rf model-collect/$collection/archives 
     157#    rm -rf model-collect/$collection/index 
     158 
     159} 
     160 
     161 
     162# copy and then svn add the collection's archives and index folders 
     163function _add_col_archives_index () { 
     164    collection=$1 
     165 
     166    #escape the filename (in case of space) and get just the basename 
     167    collection=$(get_col_basename $collection) 
     168 
     169    if [ ! -e model-collect/$collection ]; then 
     170    echo "add_col_archives_index: Adding the new collection $collection to SVN" 
     171    return; 
     172    fi 
     173 
     174    # remove the entire archives and index folders from svn 
     175    cp -r collect/$collection/archives model-collect/$collection/. 
     176    cp -r collect/$collection/index model-collect/$collection/.  
     177     
     178    svn add model-collect/$collection/archives 
     179    svn add model-collect/$collection/index 
     180} 
     181 
     182 
     183# svn update/delete a single collection 
     184function svn_process_single_collection () { 
     185    collection=$1 
     186     
     187    #escape the filename (in case of space) and get just the basename 
     188    collection=$(get_col_basename $collection) 
     189 
     190    if [ ! -e model-collect/$collection ]; then 
     191    echo "update_single_collection: Adding new collection $collection to SVN" 
    77192    return; 
    78193    fi 
     
    118233    IFS=$IFS_BAK 
    119234    IFS_BAK= 
    120  
    121     # Now svn remove what's unique to model-collect and svn add what's been rebuilt in index and archives 
    122     # see http://stackoverflow.com/questions/7502261/delete-folder-content-and-remove-from-version-control 
    123     # And doing: rm -rfv mydir/* should leave .hidden folders in mydir, see 
    124     # http://askubuntu.com/questions/72446/how-to-remove-all-files-and-subdirectories-in-a-directory-without-deleting-the-d 
    125     if [ "$mode" == "svndelete" ]; then 
    126     svn rm model-collect/$collection/archives/* 
    127     svn rm model-collect/$collection/index/* 
    128 #   rm -rfv model-collect/$collection/archives/* 
    129 #   rm -rfv model-collect/$collection/index/*    
    130     # from testing, the above will remove all the contents of archives and index  
    131     # except the .svn folder of toplevel archives/index 
    132     fi 
    133  
    134     # http://stackoverflow.com/questions/5044214/how-do-i-detect-and-or-delete-empty-subversion-directories 
    135     # http://stackoverflow.com/questions/1301203/removing-svn-files-from-all-directories 
     235} 
     236 
     237# Function that takes care of the --svnupdate flag mode of this script 
     238function update_single_collection () { 
    136239 
    137240    # copy across the contents of the rebuilt model-collection's index and archives to the svn model-collect 
     
    139242    cp -r collect/$collection/index/* model-collect/$collection/index/. 
    140243 
    141     if [ "$mode" == "svndelete" ]; then 
    142     svn add model-collect/$collection/archives/* 
    143     svn add model-collect/$collection/index/* 
    144     fi 
    145  
    146     # To undo the changes made by svndelete, run the following manually 
    147     # svn revert --depth infinity <model-collect/$collection/archives/* 
    148     # svn revert --depth infinity <model-collect/$collection/archives/* 
    149     # then remove both the local archives and index, and do an svn up to get original checkout back 
     244    echo "svn model-collect update process complete. CHECK AND COMMIT THE model-collect FOLDER!" 
    150245 
    151246    # if etc/collect.cfg is different, copy it across too? 
     
    154249 
    155250# re-build a single collection in "collect" which is a copy of model-collect 
    156 function single_collection () { 
    157     collection=$1     
    158  
    159     #escape the filename (in case of space) 
    160     collection=`echo $collection | sed 's@ @\\\ @g'` 
    161      
    162     #get just the basename 
    163     collection=`basename $collection` 
    164  
    165     # clean up empty directories: 
    166     # HASH dirs that are empty in local (non-svn) collect's archives and index/assoc,  
    167     # need to be removed from the svn in model-collect.  
    168     # They're almost empty in the mode-collect, except for the .svn folder they contain 
    169  
    170     for coldir in `find collect/$collection/archives/HASH* -type d -empty`; do 
    171     modeldir="model-$coldir" 
    172     echo "Removing empty dir from svn: $modeldir" 
    173      
    174     # remove from svn of model collect 
    175     svn rm $modeldir 
    176     rm -rf $modeldir 
    177     # remove physically from local collect (not necessary, as a rebuild will sort this out too) 
    178     rm -rf $coldir 
    179     done 
    180  
    181     for coldir in `find collect/$collection/index/assoc/HASH* -type d -empty`; do 
    182     modeldir="model-$coldir" 
    183     echo "Removing empty dir from svn: $modeldir" 
    184      
    185     # remove from svn of model collect 
    186     svn rm $modeldir 
    187     rm -rf $modeldir 
    188     # remove physically from local collect (not necessary, as a rebuild will sort this out too) 
    189     rm -rf $coldir 
    190     done 
     251function build_single_collection () { 
     252    collection=$1 
     253    
     254    collection=$(get_col_basename $collection) 
    191255     
    192256    import.pl -removeold $collection 
     
    202266} 
    203267 
    204 # unused 
    205 # re-build all collections in "collect" which is a copy of model-collect 
    206 function all_collections() { 
    207  
    208     #for each collection, import, build, move building to index 
    209     for collection in collect/*; do 
    210     single_collection $collection; 
    211     #update_single_collection $collection; 
    212     done 
    213 } 
    214268 
    215269# http://stackoverflow.com/questions/16483119/example-of-how-to-use-getopt-in-bash 
     
    277331 
    278332# If no mode provided (svndelete|svnupdate) as cmd line arg, then don't modify 
    279 # the svn model-collect folder. We leave it at rebuilding its copy in collect 
    280  
    281 #http://www.cyberciti.biz/faq/linux-unix-sleep-bash-scripting/ 
    282 #if [ "x$mode" == "x" ]; then 
    283 #    echo 
    284 #    echo "***Mode svndelete or svnupdate not provided. Defaulting to svnupdate... 3s to change to your mind" 
    285 #    echo 
    286 #    sleep 3s 
    287 #    mode=svnupdate 
    288 #fi 
     333# the svn model-collect folder. Then this script stops after rebuilding the model-copy in collect 
    289334 
    290335# the remaining arguments to the script are assumed to be collections 
     
    315360 
    316361 
    317 if test -e model-collect; then    
    318     svn up model-collect 
    319 else 
    320     svn co http://svn.greenstone.org/other-projects/nightly-tasks/diffcol/trunk/model-collect 
    321 fi 
    322  
    323362# move the existing collect folder out of the way 
    324363if [ -e collect ] && [ ! -e collect_orig ] ; then 
     
    326365fi 
    327366 
    328 # make a copy of the model-collect named as the new collect  
    329 # and remove the copy's .svn folders 
     367 
     368# get model-collect from svn 
     369# if we already have it, svn update the entire model-collect folder if processing all collections 
     370# or svn update just any collections specified in the model-collect folder 
     371if test -e model-collect; then 
     372    if [ "$1" == "" ]; then 
     373    svn up model-collect 
     374    else 
     375    for collection in "$@"; do 
     376        if [ -e model-collect/$collection ]; then 
     377        svn up model-collect/$collection 
     378        else 
     379        svn up model-collect 
     380        fi 
     381    done 
     382    fi 
     383else 
     384    svn co http://svn.greenstone.org/other-projects/nightly-tasks/diffcol/trunk/model-collect 
     385fi 
     386 
     387# Make a copy of the model-collect named as the new collect  
     388# (or if collections are specified in the cmdline arguments, copy just these over from model-collect into collect) 
     389# Then remove the copy's .svn folders 
    330390if [ -e collect_orig ]; then 
    331     if [ -e collect ]; then 
    332     rm -rf collect 
    333     fi 
    334     cp -r model-collect collect 
    335 fi 
    336  
    337 #cd collect 
    338 #find . -name ".svn" -type d -exec rm -rf {} \; 
    339 #cd .. 
    340 find collect -name ".svn" -type d -exec rm -rf {} \; #2>&1 > /dev/null 
    341  
    342  
    343 # Now delete all empty dirs in the copy of model-collect (since the dirs will not have .svn folders in them anymore) 
    344 # http://www.commandlinefu.com/commands/view/5131/recursively-remove-all-empty-directories 
    345 #find collect -type d -empty -delete 
     391    if [ ! -e collect ]; then 
     392    cp -r model-collect collect  
     393    find collect -name ".svn" -type d -exec rm -rf {} \; #2>&1 > /dev/null 
     394    else 
     395    if [ "$1" == "" ]; then 
     396        rm -rf collect 
     397        cp -r model-collect collect 
     398        find collect -name ".svn" -type d -exec rm -rf {} \; 
     399    else 
     400        for collection in "$@"; do 
     401        if [ -e collect/$collection ]; then 
     402            rm -rf collect/$collection 
     403        fi 
     404        cp -r model-collect/$collection collect/$collection 
     405        find collect/$collection -name ".svn" -type d -exec rm -rf {} \; 
     406        done 
     407    fi 
     408    fi 
     409fi 
    346410 
    347411# Set up the Greenstone environment for building 
     
    353417 
    354418if [ "$1" == "" ]; then 
    355     # all_collections    
     419 
     420    # all_collections    
    356421    #for each collection, import, build, move building to index 
    357422    for collection in collect/*; do 
    358     single_collection $collection; 
     423    build_single_collection $collection; 
     424 
    359425    if [ "x$mode" != "x" ]; then 
    360         # Now delete all empty dirs in the copy of model-collect (since the dirs will not have .svn folders in them anymore) 
    361             # http://www.commandlinefu.com/commands/view/5131/recursively-remove-all-empty-directories 
    362 #       find collect/$collection/archives/HASH* -type d -empty -delete 
    363 #       find collect/$collection/index/assoc/HASH* -type d -empty -delete 
    364         update_single_collection $collection $mode; 
     426        svn_process_single_collection $collection 
     427 
     428        if [ "x$mode" == "xsvnupdate" ]; then 
     429        update_single_collection $collection 
     430        fi 
    365431    fi 
    366432    done 
     433 
     434    # having rebuilt all the collections, just the processing for svndelete/update remains: 
     435    if [ "x$mode" == "xsvndelete" ]; then 
     436    svn_delete 
     437    fi 
     438 
    367439else 
    368440    # Command-line args are a list of collections,  
     
    372444    collection=collect/$collection 
    373445    if test -e $collection; then    
    374         single_collection $collection; 
     446        build_single_collection $collection; 
     447 
    375448        if [ "x$mode" != "x" ]; then 
    376 #       find collect/$collection/archives/HASH* -type d -empty -delete 
    377 #       find collect/$collection/index/assoc/HASH* -type d -empty -delete 
    378         update_single_collection $collection $mode; 
     449        svn_process_single_collection $collection 
     450 
     451        if [ "x$mode" != "x" ]; then 
     452            update_single_collection $collection 
     453        fi 
    379454        fi 
    380455    else 
     
    382457    fi 
    383458    done 
     459 
     460    # having rebuilt the specified collections above, just the processing for svndelete/update remains 
     461    if [ "x$mode" == "xsvndelete" ]; then 
     462    svn_delete $@ 
     463    fi 
     464 
    384465fi 
    385466 
     
    405486 
    406487 
    407 # The following in a separate script file will delete all folders from model-collect that are 
     488# deletes empty dirs 
     489#       find collect/$collection/archives/HASH* -type d -empty -delete 
     490#       find collect/$collection/index/assoc/HASH* -type d -empty -delete 
     491 
     492# To recursively delete all empty dirs in the copy of model-collect (since the dirs will not have .svn folders in them anymore) 
     493# http://www.commandlinefu.com/commands/view/5131/recursively-remove-all-empty-directories 
     494#find collect -type d -empty -delete 
     495 
     496# The following when put in a separate script file will delete all folders from model-collect that are 
    408497# empty in the copied collection (all folders which contain only a .svn subfolder in model-collect) 
     498#!/bin/bash 
    409499 
    410500#for collection in collect/*; do