Index: other-projects/nightly-tasks/diffcol/trunk/gen-model-colls.sh
===================================================================
--- other-projects/nightly-tasks/diffcol/trunk/gen-model-colls.sh	(revision 28027)
+++ other-projects/nightly-tasks/diffcol/trunk/gen-model-colls.sh	(revision 28037)
@@ -12,9 +12,42 @@
 
 # USAGE
-# Put this file in the toplevel of the Greenstone 2 binary/SVN installation 
+# Put this file in the toplevel of the Greenstone 2 binary/compiled SVN installation
 # that you want to generate the model collections with.
-# Call with 0 args to process all model collections
-# Call with n args, where each is a collection name in the model-collections
-# list
+# You can provide a list of collection names or none, in which case all the collections
+# are processed.
+
+# Pass in --svnupdate to copy across the contents of archives and index in the
+# rebuilt collection, overwriting their equivalents in the svn model collection,
+# but not removing any extraneous HASH folders already present.
+
+# Pass in --svndelete to remove what's in archives and index from svn and replace
+# the contents of archives and index with the matching contents from the rebuild 
+# collection. Useful for when the HASH directory naming has changed and everything
+# in archives and index has to be wiped out and moved back in from the rebuilt col.
+
+# If neither is passed in, then the collections are rebuilt but the svn model-collect
+# is not updated and the repository is not updated.
+
+# This program leaves you to do the final commit on the (svn) model-collect folder!
+
+# Examples of usage:
+# ./gen-model-colls.sh
+# ./gen-model-colls.sh --svndelete
+# ./gen-model-colls.sh --svnupdate Tudor-Basic Tudor-Enhanced
+
+# The first just rebuilds all the collections in a new folder called collect and stops there
+# The second rebuilds all the collections and then removes all contents of the archives and the
+# index folders from the svn checked-out model-collect and removes them from svn. Then it copies 
+# across all the contents of the rebuilt archives and index into model-collect and svn adds them.
+# The third example checks out all the model-collections again, but rebuilds only the 3 collections 
+# specified in the new collect folder. Then it copies across the contents of the archives and index
+# folders of those 3 collections into their model-collect equivalents.
+
+# You then still have to do the final svn commit on the model-collect folder.
+
+# Also valid examples:
+# ./gen-model-colls.sh Tudor-Basic Tudor-Enhanced
+# ./gen-model-colls.sh --svndelete Tudor-Basic Tudor-Enhanced
+# ./gen-model-colls.sh --svnupdate
 
 # PSEUDOCODE
@@ -23,7 +56,92 @@
 # Makes a copy
 # In the copy: gets rid of their .svn folders, and builds each collection in turn, moving building to index once done
-
-
-
+# If --svndelete was passed in: svn removes model-collect/archives/* and model-collect/index/*, copies over index/* 
+# and archives/* from collect into model-collect and svn adds model-collect/archives/* and model-collect/index/*
+# If --svnupdate was passed in: copies collect/archives/* into model-collect/archives/*, and copies collect/index/*
+# into model-collect/index/*, overwriting files that already existed but have now been updated upon rebuild. However, 
+# --svnupdate will leave untouched any files and folders unique to model-collect.
+
+# svn update/delete a single collection
+function update_single_collection () {
+    collection=$1
+    mode=$2
+
+    #escape the filename (in case of space)
+    collection=`echo $collection | sed 's@ @\\\ @g'`
+    
+    #get just the basename
+    collection=`basename $collection`
+
+    if [ ! -e model-collect/$collection ]; then
+	echo "$collection does not exist in model-collect, commit separately"
+	return;
+    fi
+
+    # diff the svn model and rebuilt model collections
+    diff_result=`diff -rq model-collect/$collection collect/$collection | grep -v ".svn"`
+#    echo "Diff result for collection $collection: $diff_result"
+
+    # if no differences in the current collection, then we're done
+    if [ "x$diff_result" == "x" ]; then
+	echo "No differences in collection $collection"
+	return;
+    fi
+
+    # check that none of the lines mention files outside the archives or index folders
+    # http://en.gibney.org/tell_the_bash_to_split_by_newline_charac
+    # http://forums.gentoo.org/viewtopic-p-3130541.html
+
+    # http://wi-fizzle.com/article/276
+    # http://stackoverflow.com/questions/918886/how-do-i-split-a-string-on-a-delimiter-in-bash
+    # http://www.linuxquestions.org/questions/programming-9/split-a-string-on-newlines-bash-313206/
+    # http://unix.stackexchange.com/questions/39473/command-substitution-splitting-on-newline-but-not-space
+
+    # store backup of Internal Field Separator value, then set IFS to newline for splitting on newline
+
+    IFS_BAK=$IFS
+#    IFS='\n' # splits on all whitespace
+IFS='
+'
+    # in the lines returned from the diff, test for archives or newline
+    # http://stackoverflow.com/questions/229551/string-contains-in-bash
+    for line in `diff -rq model-collect/$collection collect/$collection | grep -v ".svn"`; do
+	# echo "LINE: $line"	
+	if [[ "$line" != *archives* && "$line" != *index* ]]; then
+	    # the file that is different is neither in index nor in archives, send this diffline to the report
+	    echo $line >> report.txt
+	fi
+    done
+ 
+    IFS=$IFS_BAK
+    IFS_BAK=
+
+    # Now svn remove what's unique to model-collect and svn add what's been rebuilt in index and archives
+    # see # http://stackoverflow.com/questions/7502261/delete-folder-content-and-remove-from-version-control
+ 
+    if [ "$mode" == "svndelete" ]; then
+	svn rm model-collect/$collection/archives/*
+	svn rm model-collect/$collection/index/*
+    fi
+
+    # copy across the contents of the rebuilt model-collection's index and archives to the svn model-collect
+    cp -r collect/$collection/archives/* model-collect/$collection/archives/.
+    cp -r collect/$collection/index/* model-collect/$collection/index/.
+
+    if [ "$mode" == "svndelete" ]; then
+	svn add model-collect/$collection/archives/*
+	svn add model-collect/$collection/index/*
+    fi
+
+    # To undo the changes made by svndelete, run the following manually
+    # svn revert --depth infinity <model-collect/$collection/archives/*
+    # svn revert --depth infinity <model-collect/$collection/archives/*
+    # then remove both the local archives and index, and do an svn up to get original checkout back
+
+    # if etc/collect.cfg is different, copy it across too?
+
+
+}
+
+# re-build a single collection in "collect" which is a copy of model-collect
 function single_collection () {
     collection=$1    
@@ -47,5 +165,6 @@
 }
 
-
+# unused
+# re-build all collections in "collect" which is a copy of model-collect
 function all_collections() {
 
@@ -53,9 +172,98 @@
     for collection in collect/*; do
 	single_collection $collection;
+	#update_single_collection $collection;
     done
 }
 
+# http://stackoverflow.com/questions/16483119/example-of-how-to-use-getopt-in-bash
+function usage() { 
+# usage() { echo "Usage: $0 [-s <45|90>] [-p <string>]" 1>&2; exit 1; }
+
+    echo "*******************************************"
+    echo "Usage: $0 [--svnupdate|--svndelete] [col1, col2, col3,...]"; 
+    echo "If no collections are provided, all collections will be processed.";
+    echo "If neither svnupdate nor svndelete are provided, svnupdate is assumed.";
+    echo "*******************************************"
+    exit 1; 
+}
+
 
 # The program starts here
+
+# process optional command line arguments
+# http://blog.onetechnical.com/2012/07/16/bash-getopt-versus-getopts/
+# Execute getopt
+ARGS=$(getopt -o ud -l "svnupdate,svndelete" -n "$0" -- "$@");
+
+#Bad arguments
+if [ $? -ne 0 ];then
+    usage
+    exit 1
+fi
+
+eval set -- "$ARGS";
+
+# mode can be svndelete or svnupdate
+mode=
+
+# -n: http://tldp.org/LDP/abs/html/testconstructs.html
+while true; do
+  case "$1" in
+    -d|--svndelete)
+      shift;
+      if [ "x$mode" == "xsvnupdate" ]; then
+	  echo
+	  echo "Can't use both svndelete and svnupdate"
+	  usage
+	  exit 1
+      else
+	  mode=svndelete
+      fi
+      ;;
+    -u|--svnupdate)
+      shift;
+      if [ "x$mode" == "xsvndelete" ]; then
+	  echo
+	  echo "Can't use both svndelete and svnupdate"
+	  usage
+	  exit 1
+      else
+	  mode=svnupdate
+      fi
+      ;;
+    --)
+      shift;
+      break;
+      ;;
+  esac
+done
+
+# If no mode provided (svndelete|svnupdate) as cmd line arg, then don't modify
+# the svn model-collect folder. We leave it at rebuilding its copy in collect
+
+#http://www.cyberciti.biz/faq/linux-unix-sleep-bash-scripting/
+#if [ "x$mode" == "x" ]; then
+#    echo
+#    echo "***Mode svndelete or svnupdate not provided. Defaulting to svnupdate... 3s to change to your mind"
+#    echo
+#    sleep 3s
+#    mode=svnupdate
+#fi
+
+# the remaining arguments to the script are assumed to be collections
+
+# debugging
+#for collection in "$@"; do
+#    collection=collect/$collection
+#    echo "Collection: $collection"
+#done
+
+# finished processing arguments
+
+
+# report will contain the output of the diff for
+if [ -f report.txt ]; then
+    rm report.txt
+fi
 
 # Need pdfbox for the PDFBox tutorial
@@ -93,5 +301,5 @@
 #find . -name ".svn" -type d -exec rm -rf {} \;
 #cd ..
-find collect -name ".svn" -type d -exec rm -rf {} \;
+find collect -name ".svn" -type d -exec rm -rf {} \; #2>&1 > /dev/null
 
 
@@ -104,12 +312,23 @@
 
 if [ "$1" == "" ]; then
-	all_collections
+	# all_collections	
+    #for each collection, import, build, move building to index
+    for collection in collect/*; do
+	single_collection $collection;
+	if [ "x$mode" != "x" ]; then
+	    update_single_collection $collection $mode;
+	fi
+    done
 else
     # Command-line args are a list of collections, 
     # process each command-line arg, after confirming such a collection exists
 
-    for collection in collect/"$@"; do
+    for collection in "$@"; do
+	collection=collect/$collection
 	if test -e $collection; then   
 	    single_collection $collection;
+	    if [ "x$mode" != "x" ]; then
+		update_single_collection $collection $mode;
+	    fi
 	else
 	    echo "Can't find collection $collection. Skipping."
@@ -117,2 +336,24 @@
     done
 fi
+
+
+echo
+echo "*****************************************"
+# if we were svn updating/deleting collections, then mode was set
+# if in that case a report was generated with additional differences, point the user to it
+if [ -f report.txt ] && [ "x$mode" != "x" ]; then
+    echo "Some files outside of archives and index folders were different. See report.txt"
+    echo
+fi
+
+echo "The original collect directory has been left renamed as collect_orig"
+echo
+if [ "x$mode" == "x" ]; then
+    echo "The model-collect folder has not been altered. Changes have only been made to collect"
+else
+    echo "You still need to run svn status and svn commit on the model-collect folder"
+fi
+echo "*****************************************"
+echo
+
+
