Changeset 28048 for other-projects/nightly-tasks
- Timestamp:
- 2013-08-14T22:02:17+12:00 (11 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
other-projects/nightly-tasks/diffcol/trunk/gen-model-colls.sh
r28046 r28048 62 62 # --svnupdate will leave untouched any files and folders unique to model-collect. 63 63 64 # svn update/delete a single collection 65 function update_single_collection () { 64 # See earlier version of this script: 65 # To svn remove what's unique to model-collect and svn add what's been rebuilt in index and archives 66 # see http://stackoverflow.com/questions/7502261/delete-folder-content-and-remove-from-version-control 67 68 # http://stackoverflow.com/questions/5044214/how-do-i-detect-and-or-delete-empty-subversion-directories 69 # http://stackoverflow.com/questions/1301203/removing-svn-files-from-all-directories 70 71 72 # DON'T ADD ANY FURTHER ECHO STATEMENTS IN FUNCTION get_col_basename 73 # "you have to be really careful on what you have in this function, as having any code which will eventually echo will mean that you get incorrect return string." 74 # see http://stackoverflow.com/questions/3236871/how-to-return-a-string-value-from-a-bash-function 75 function get_col_basename () { 66 76 collection=$1 67 mode=$268 77 69 78 #escape the filename (in case of space) … … 73 82 collection=`basename $collection` 74 83 84 # returning a string does not work in bash 85 # see http://stackoverflow.com/questions/3236871/how-to-return-a-string-value-from-a-bash-function 86 87 #return $collection 88 echo $collection 89 } 90 91 92 # Function that handles the --svndelete flag (mode) of this script 93 function svn_delete () { 94 95 # svn remove archives and index in each collection 96 # commit them all 97 # copy over newly rebuilt archives and index into each model-collection 98 # svn add the new archives and index folders of each collection 99 # commit them all 100 101 102 if [ "x$1" == "x" ]; then 103 for collection in collect/*; do 104 _del_col_archives_index $collection 105 done 106 else 107 for collection in "$@"; do 108 _del_col_archives_index $collection 109 done 110 fi 111 112 # commit all the svn rm statements done above in one go: 113 # don't do `svn up` here, as this will then retrieve all the folders that were svn-removed 114 svn commit -m "Clean rebuild of model collections 1/2. Clearing out deprecated archives and index." model-collect 115 116 # do an svn up to locally delete what was svn-removed above, BEFORE copying from the rebuilt archives and index folders 117 svn up model-collect 118 119 if [ "x$1" == "x" ]; then 120 for collection in collect/*; do 121 _add_col_archives_index $collection 122 done 123 else 124 for collection in "$@"; do 125 _add_col_archives_index $collection 126 done 127 fi 128 129 # commit all the svn add statements done just above in one go 130 svn commit -m "Clean rebuild of model collections 2/2. Adding rebuilt archives and index." model-collect 131 } 132 133 # To undo the changes made by svndelete, run the following manually 134 # svn revert --depth infinity <model-collect/$collection/archives/* 135 # svn revert --depth infinity <model-collect/$collection/archives/* 136 # then remove both the local archives and index, and do an svn up to get original checkout back 137 138 # svn delete this collection's archives and index folders 139 # (The commit will be done when in one step for all collections on which this function was called) 140 function _del_col_archives_index () { 141 collection=$1 142 143 #escape the filename (in case of space) and get just the basename 144 collection=$(get_col_basename $collection) 145 75 146 if [ ! -e model-collect/$collection ]; then 76 echo "$collection does not exist in model-collect, commit separately" 147 echo "del_col_archives_index: $collection does not exist in model-collect, will svn add this new collection shortly" 148 return; 149 fi 150 151 # remove the entire archives and index folders from svn 152 svn rm model-collect/$collection/archives 153 svn rm model-collect/$collection/index 154 155 # for TESTING purposes: 156 # rm -rf model-collect/$collection/archives 157 # rm -rf model-collect/$collection/index 158 159 } 160 161 162 # copy and then svn add the collection's archives and index folders 163 function _add_col_archives_index () { 164 collection=$1 165 166 #escape the filename (in case of space) and get just the basename 167 collection=$(get_col_basename $collection) 168 169 if [ ! -e model-collect/$collection ]; then 170 echo "add_col_archives_index: Adding the new collection $collection to SVN" 171 return; 172 fi 173 174 # remove the entire archives and index folders from svn 175 cp -r collect/$collection/archives model-collect/$collection/. 176 cp -r collect/$collection/index model-collect/$collection/. 177 178 svn add model-collect/$collection/archives 179 svn add model-collect/$collection/index 180 } 181 182 183 # svn update/delete a single collection 184 function svn_process_single_collection () { 185 collection=$1 186 187 #escape the filename (in case of space) and get just the basename 188 collection=$(get_col_basename $collection) 189 190 if [ ! -e model-collect/$collection ]; then 191 echo "update_single_collection: Adding new collection $collection to SVN" 77 192 return; 78 193 fi … … 118 233 IFS=$IFS_BAK 119 234 IFS_BAK= 120 121 # Now svn remove what's unique to model-collect and svn add what's been rebuilt in index and archives 122 # see http://stackoverflow.com/questions/7502261/delete-folder-content-and-remove-from-version-control 123 # And doing: rm -rfv mydir/* should leave .hidden folders in mydir, see 124 # http://askubuntu.com/questions/72446/how-to-remove-all-files-and-subdirectories-in-a-directory-without-deleting-the-d 125 if [ "$mode" == "svndelete" ]; then 126 svn rm model-collect/$collection/archives/* 127 svn rm model-collect/$collection/index/* 128 # rm -rfv model-collect/$collection/archives/* 129 # rm -rfv model-collect/$collection/index/* 130 # from testing, the above will remove all the contents of archives and index 131 # except the .svn folder of toplevel archives/index 132 fi 133 134 # http://stackoverflow.com/questions/5044214/how-do-i-detect-and-or-delete-empty-subversion-directories 135 # http://stackoverflow.com/questions/1301203/removing-svn-files-from-all-directories 235 } 236 237 # Function that takes care of the --svnupdate flag mode of this script 238 function update_single_collection () { 136 239 137 240 # copy across the contents of the rebuilt model-collection's index and archives to the svn model-collect … … 139 242 cp -r collect/$collection/index/* model-collect/$collection/index/. 140 243 141 if [ "$mode" == "svndelete" ]; then 142 svn add model-collect/$collection/archives/* 143 svn add model-collect/$collection/index/* 144 fi 145 146 # To undo the changes made by svndelete, run the following manually 147 # svn revert --depth infinity <model-collect/$collection/archives/* 148 # svn revert --depth infinity <model-collect/$collection/archives/* 149 # then remove both the local archives and index, and do an svn up to get original checkout back 244 echo "svn model-collect update process complete. CHECK AND COMMIT THE model-collect FOLDER!" 150 245 151 246 # if etc/collect.cfg is different, copy it across too? … … 154 249 155 250 # re-build a single collection in "collect" which is a copy of model-collect 156 function single_collection () { 157 collection=$1 158 159 #escape the filename (in case of space) 160 collection=`echo $collection | sed 's@ @\\\ @g'` 161 162 #get just the basename 163 collection=`basename $collection` 164 165 # clean up empty directories: 166 # HASH dirs that are empty in local (non-svn) collect's archives and index/assoc, 167 # need to be removed from the svn in model-collect. 168 # They're almost empty in the mode-collect, except for the .svn folder they contain 169 170 for coldir in `find collect/$collection/archives/HASH* -type d -empty`; do 171 modeldir="model-$coldir" 172 echo "Removing empty dir from svn: $modeldir" 173 174 # remove from svn of model collect 175 svn rm $modeldir 176 rm -rf $modeldir 177 # remove physically from local collect (not necessary, as a rebuild will sort this out too) 178 rm -rf $coldir 179 done 180 181 for coldir in `find collect/$collection/index/assoc/HASH* -type d -empty`; do 182 modeldir="model-$coldir" 183 echo "Removing empty dir from svn: $modeldir" 184 185 # remove from svn of model collect 186 svn rm $modeldir 187 rm -rf $modeldir 188 # remove physically from local collect (not necessary, as a rebuild will sort this out too) 189 rm -rf $coldir 190 done 251 function build_single_collection () { 252 collection=$1 253 254 collection=$(get_col_basename $collection) 191 255 192 256 import.pl -removeold $collection … … 202 266 } 203 267 204 # unused205 # re-build all collections in "collect" which is a copy of model-collect206 function all_collections() {207 208 #for each collection, import, build, move building to index209 for collection in collect/*; do210 single_collection $collection;211 #update_single_collection $collection;212 done213 }214 268 215 269 # http://stackoverflow.com/questions/16483119/example-of-how-to-use-getopt-in-bash … … 277 331 278 332 # If no mode provided (svndelete|svnupdate) as cmd line arg, then don't modify 279 # the svn model-collect folder. We leave it at rebuilding its copy in collect 280 281 #http://www.cyberciti.biz/faq/linux-unix-sleep-bash-scripting/ 282 #if [ "x$mode" == "x" ]; then 283 # echo 284 # echo "***Mode svndelete or svnupdate not provided. Defaulting to svnupdate... 3s to change to your mind" 285 # echo 286 # sleep 3s 287 # mode=svnupdate 288 #fi 333 # the svn model-collect folder. Then this script stops after rebuilding the model-copy in collect 289 334 290 335 # the remaining arguments to the script are assumed to be collections … … 315 360 316 361 317 if test -e model-collect; then318 svn up model-collect319 else320 svn co http://svn.greenstone.org/other-projects/nightly-tasks/diffcol/trunk/model-collect321 fi322 323 362 # move the existing collect folder out of the way 324 363 if [ -e collect ] && [ ! -e collect_orig ] ; then … … 326 365 fi 327 366 328 # make a copy of the model-collect named as the new collect 329 # and remove the copy's .svn folders 367 368 # get model-collect from svn 369 # if we already have it, svn update the entire model-collect folder if processing all collections 370 # or svn update just any collections specified in the model-collect folder 371 if test -e model-collect; then 372 if [ "$1" == "" ]; then 373 svn up model-collect 374 else 375 for collection in "$@"; do 376 if [ -e model-collect/$collection ]; then 377 svn up model-collect/$collection 378 else 379 svn up model-collect 380 fi 381 done 382 fi 383 else 384 svn co http://svn.greenstone.org/other-projects/nightly-tasks/diffcol/trunk/model-collect 385 fi 386 387 # Make a copy of the model-collect named as the new collect 388 # (or if collections are specified in the cmdline arguments, copy just these over from model-collect into collect) 389 # Then remove the copy's .svn folders 330 390 if [ -e collect_orig ]; then 331 if [ -e collect ]; then 332 rm -rf collect 333 fi 334 cp -r model-collect collect 335 fi 336 337 #cd collect 338 #find . -name ".svn" -type d -exec rm -rf {} \; 339 #cd .. 340 find collect -name ".svn" -type d -exec rm -rf {} \; #2>&1 > /dev/null 341 342 343 # Now delete all empty dirs in the copy of model-collect (since the dirs will not have .svn folders in them anymore) 344 # http://www.commandlinefu.com/commands/view/5131/recursively-remove-all-empty-directories 345 #find collect -type d -empty -delete 391 if [ ! -e collect ]; then 392 cp -r model-collect collect 393 find collect -name ".svn" -type d -exec rm -rf {} \; #2>&1 > /dev/null 394 else 395 if [ "$1" == "" ]; then 396 rm -rf collect 397 cp -r model-collect collect 398 find collect -name ".svn" -type d -exec rm -rf {} \; 399 else 400 for collection in "$@"; do 401 if [ -e collect/$collection ]; then 402 rm -rf collect/$collection 403 fi 404 cp -r model-collect/$collection collect/$collection 405 find collect/$collection -name ".svn" -type d -exec rm -rf {} \; 406 done 407 fi 408 fi 409 fi 346 410 347 411 # Set up the Greenstone environment for building … … 353 417 354 418 if [ "$1" == "" ]; then 355 # all_collections 419 420 # all_collections 356 421 #for each collection, import, build, move building to index 357 422 for collection in collect/*; do 358 single_collection $collection; 423 build_single_collection $collection; 424 359 425 if [ "x$mode" != "x" ]; then 360 # Now delete all empty dirs in the copy of model-collect (since the dirs will not have .svn folders in them anymore)361 # http://www.commandlinefu.com/commands/view/5131/recursively-remove-all-empty-directories 362 # find collect/$collection/archives/HASH* -type d -empty -delete 363 # find collect/$collection/index/assoc/HASH* -type d -empty -delete 364 update_single_collection $collection $mode;426 svn_process_single_collection $collection 427 428 if [ "x$mode" == "xsvnupdate" ]; then 429 update_single_collection $collection 430 fi 365 431 fi 366 432 done 433 434 # having rebuilt all the collections, just the processing for svndelete/update remains: 435 if [ "x$mode" == "xsvndelete" ]; then 436 svn_delete 437 fi 438 367 439 else 368 440 # Command-line args are a list of collections, … … 372 444 collection=collect/$collection 373 445 if test -e $collection; then 374 single_collection $collection; 446 build_single_collection $collection; 447 375 448 if [ "x$mode" != "x" ]; then 376 # find collect/$collection/archives/HASH* -type d -empty -delete 377 # find collect/$collection/index/assoc/HASH* -type d -empty -delete 378 update_single_collection $collection $mode; 449 svn_process_single_collection $collection 450 451 if [ "x$mode" != "x" ]; then 452 update_single_collection $collection 453 fi 379 454 fi 380 455 else … … 382 457 fi 383 458 done 459 460 # having rebuilt the specified collections above, just the processing for svndelete/update remains 461 if [ "x$mode" == "xsvndelete" ]; then 462 svn_delete $@ 463 fi 464 384 465 fi 385 466 … … 405 486 406 487 407 # The following in a separate script file will delete all folders from model-collect that are 488 # deletes empty dirs 489 # find collect/$collection/archives/HASH* -type d -empty -delete 490 # find collect/$collection/index/assoc/HASH* -type d -empty -delete 491 492 # To recursively delete all empty dirs in the copy of model-collect (since the dirs will not have .svn folders in them anymore) 493 # http://www.commandlinefu.com/commands/view/5131/recursively-remove-all-empty-directories 494 #find collect -type d -empty -delete 495 496 # The following when put in a separate script file will delete all folders from model-collect that are 408 497 # empty in the copied collection (all folders which contain only a .svn subfolder in model-collect) 498 #!/bin/bash 409 499 410 500 #for collection in collect/*; do
Note:
See TracChangeset
for help on using the changeset viewer.
