Index: other-projects/nightly-tasks/diffcol/trunk/gen-model-colls.sh
===================================================================
--- other-projects/nightly-tasks/diffcol/trunk/gen-model-colls.sh	(revision 28047)
+++ other-projects/nightly-tasks/diffcol/trunk/gen-model-colls.sh	(revision 28048)
@@ -62,8 +62,17 @@
 # --svnupdate will leave untouched any files and folders unique to model-collect.
 
-# svn update/delete a single collection
-function update_single_collection () {
+# See earlier version of this script:
+# To svn remove what's unique to model-collect and svn add what's been rebuilt in index and archives
+# see http://stackoverflow.com/questions/7502261/delete-folder-content-and-remove-from-version-control
+
+# http://stackoverflow.com/questions/5044214/how-do-i-detect-and-or-delete-empty-subversion-directories
+# http://stackoverflow.com/questions/1301203/removing-svn-files-from-all-directories
+
+
+# DON'T ADD ANY FURTHER ECHO STATEMENTS IN FUNCTION get_col_basename
+# "you have to be really careful on what you have in this function, as having any code which will eventually echo will mean that you get incorrect return string."
+# see http://stackoverflow.com/questions/3236871/how-to-return-a-string-value-from-a-bash-function
+function get_col_basename () {
     collection=$1
-    mode=$2
 
     #escape the filename (in case of space)
@@ -73,6 +82,112 @@
     collection=`basename $collection`
 
+    # returning a string does not work in bash
+    # see http://stackoverflow.com/questions/3236871/how-to-return-a-string-value-from-a-bash-function    
+
+    #return $collection
+    echo $collection
+}
+
+
+# Function that handles the --svndelete flag (mode) of this script
+function svn_delete () {
+
+    # svn remove archives and index in each collection
+    # commit them all
+    # copy over newly rebuilt archives and index into each model-collection
+    # svn add the new archives and index folders of each collection
+    # commit them all
+    
+    
+    if [ "x$1" == "x" ]; then
+	for collection in collect/*; do	    
+	    _del_col_archives_index $collection
+	done
+    else 
+	for collection in "$@"; do
+	    _del_col_archives_index $collection
+	done
+    fi
+
+    # commit all the svn rm statements done above in one go:
+    # don't do `svn up` here, as this will then retrieve all the folders that were svn-removed
+    svn commit -m "Clean rebuild of model collections 1/2. Clearing out deprecated archives and index." model-collect
+
+    # do an svn up to locally delete what was svn-removed above, BEFORE copying from the rebuilt archives and index folders
+    svn up model-collect
+
+    if [ "x$1" == "x" ]; then
+	for collection in collect/*; do	    
+	    _add_col_archives_index $collection
+	done
+    else 
+	for collection in "$@"; do
+	    _add_col_archives_index $collection
+	done
+    fi
+
+    # commit all the svn add statements done just above in one go
+    svn commit -m "Clean rebuild of model collections 2/2. Adding rebuilt archives and index." model-collect
+}
+
+# To undo the changes made by svndelete, run the following manually
+# svn revert --depth infinity <model-collect/$collection/archives/*
+# svn revert --depth infinity <model-collect/$collection/archives/*
+# then remove both the local archives and index, and do an svn up to get original checkout back
+
+# svn delete this collection's archives and index folders
+# (The commit will be done when in one step for all collections on which this function was called)
+function _del_col_archives_index () {
+    collection=$1
+
+    #escape the filename (in case of space) and get just the basename
+    collection=$(get_col_basename $collection)
+
     if [ ! -e model-collect/$collection ]; then
-	echo "$collection does not exist in model-collect, commit separately"
+	echo "del_col_archives_index: $collection does not exist in model-collect, will svn add this new collection shortly"
+	return;
+    fi
+
+    # remove the entire archives and index folders from svn
+    svn rm model-collect/$collection/archives
+    svn rm model-collect/$collection/index
+
+# for TESTING purposes:
+#    rm -rf model-collect/$collection/archives
+#    rm -rf model-collect/$collection/index
+
+}
+
+
+# copy and then svn add the collection's archives and index folders
+function _add_col_archives_index () {
+    collection=$1
+
+    #escape the filename (in case of space) and get just the basename
+    collection=$(get_col_basename $collection)
+
+    if [ ! -e model-collect/$collection ]; then
+	echo "add_col_archives_index: Adding the new collection $collection to SVN"
+	return;
+    fi
+
+    # remove the entire archives and index folders from svn
+    cp -r collect/$collection/archives model-collect/$collection/.
+    cp -r collect/$collection/index model-collect/$collection/.	
+    
+    svn add model-collect/$collection/archives
+    svn add model-collect/$collection/index
+}
+
+
+# svn update/delete a single collection
+function svn_process_single_collection () {
+    collection=$1
+    
+    #escape the filename (in case of space) and get just the basename
+    collection=$(get_col_basename $collection)
+
+    if [ ! -e model-collect/$collection ]; then
+	echo "update_single_collection: Adding new collection $collection to SVN"
 	return;
     fi
@@ -118,20 +233,8 @@
     IFS=$IFS_BAK
     IFS_BAK=
-
-    # Now svn remove what's unique to model-collect and svn add what's been rebuilt in index and archives
-    # see http://stackoverflow.com/questions/7502261/delete-folder-content-and-remove-from-version-control
-    # And doing: rm -rfv mydir/* should leave .hidden folders in mydir, see
-    # http://askubuntu.com/questions/72446/how-to-remove-all-files-and-subdirectories-in-a-directory-without-deleting-the-d
-    if [ "$mode" == "svndelete" ]; then
-	svn rm model-collect/$collection/archives/*
-	svn rm model-collect/$collection/index/*
-#	rm -rfv model-collect/$collection/archives/*
-#	rm -rfv model-collect/$collection/index/*	
-	# from testing, the above will remove all the contents of archives and index 
-	# except the .svn folder of toplevel archives/index
-    fi
-
-    # http://stackoverflow.com/questions/5044214/how-do-i-detect-and-or-delete-empty-subversion-directories
-    # http://stackoverflow.com/questions/1301203/removing-svn-files-from-all-directories
+}
+
+# Function that takes care of the --svnupdate flag mode of this script
+function update_single_collection () {
 
     # copy across the contents of the rebuilt model-collection's index and archives to the svn model-collect
@@ -139,13 +242,5 @@
     cp -r collect/$collection/index/* model-collect/$collection/index/.
 
-    if [ "$mode" == "svndelete" ]; then
-	svn add model-collect/$collection/archives/*
-	svn add model-collect/$collection/index/*
-    fi
-
-    # To undo the changes made by svndelete, run the following manually
-    # svn revert --depth infinity <model-collect/$collection/archives/*
-    # svn revert --depth infinity <model-collect/$collection/archives/*
-    # then remove both the local archives and index, and do an svn up to get original checkout back
+    echo "svn model-collect update process complete. CHECK AND COMMIT THE model-collect FOLDER!"
 
     # if etc/collect.cfg is different, copy it across too?
@@ -154,39 +249,8 @@
 
 # re-build a single collection in "collect" which is a copy of model-collect
-function single_collection () {
-    collection=$1    
-
-    #escape the filename (in case of space)
-    collection=`echo $collection | sed 's@ @\\\ @g'`
-    
-    #get just the basename
-    collection=`basename $collection`
-
-    # clean up empty directories:
-    # HASH dirs that are empty in local (non-svn) collect's archives and index/assoc, 
-    # need to be removed from the svn in model-collect. 
-    # They're almost empty in the mode-collect, except for the .svn folder they contain
-
-    for coldir in `find collect/$collection/archives/HASH* -type d -empty`; do
-	modeldir="model-$coldir"
-	echo "Removing empty dir from svn: $modeldir"
-	
-	# remove from svn of model collect
-	svn rm $modeldir
-	rm -rf $modeldir
-	# remove physically from local collect (not necessary, as a rebuild will sort this out too)
-	rm -rf $coldir
-    done
-
-    for coldir in `find collect/$collection/index/assoc/HASH* -type d -empty`; do
-	modeldir="model-$coldir"
-	echo "Removing empty dir from svn: $modeldir"
-	
-	# remove from svn of model collect
-	svn rm $modeldir
-	rm -rf $modeldir
-	# remove physically from local collect (not necessary, as a rebuild will sort this out too)
-	rm -rf $coldir
-    done
+function build_single_collection () {
+    collection=$1
+   
+    collection=$(get_col_basename $collection)
     
     import.pl -removeold $collection
@@ -202,14 +266,4 @@
 }
 
-# unused
-# re-build all collections in "collect" which is a copy of model-collect
-function all_collections() {
-
-    #for each collection, import, build, move building to index
-    for collection in collect/*; do
-	single_collection $collection;
-	#update_single_collection $collection;
-    done
-}
 
 # http://stackoverflow.com/questions/16483119/example-of-how-to-use-getopt-in-bash
@@ -277,14 +331,5 @@
 
 # If no mode provided (svndelete|svnupdate) as cmd line arg, then don't modify
-# the svn model-collect folder. We leave it at rebuilding its copy in collect
-
-#http://www.cyberciti.biz/faq/linux-unix-sleep-bash-scripting/
-#if [ "x$mode" == "x" ]; then
-#    echo
-#    echo "***Mode svndelete or svnupdate not provided. Defaulting to svnupdate... 3s to change to your mind"
-#    echo
-#    sleep 3s
-#    mode=svnupdate
-#fi
+# the svn model-collect folder. Then this script stops after rebuilding the model-copy in collect
 
 # the remaining arguments to the script are assumed to be collections
@@ -315,10 +360,4 @@
 
 
-if test -e model-collect; then   
-    svn up model-collect
-else
-    svn co http://svn.greenstone.org/other-projects/nightly-tasks/diffcol/trunk/model-collect
-fi
-
 # move the existing collect folder out of the way
 if [ -e collect ] && [ ! -e collect_orig ] ; then
@@ -326,22 +365,47 @@
 fi
 
-# make a copy of the model-collect named as the new collect 
-# and remove the copy's .svn folders
+
+# get model-collect from svn
+# if we already have it, svn update the entire model-collect folder if processing all collections
+# or svn update just any collections specified in the model-collect folder
+if test -e model-collect; then
+    if [ "$1" == "" ]; then
+	svn up model-collect
+    else
+	for collection in "$@"; do
+	    if [ -e model-collect/$collection ]; then
+		svn up model-collect/$collection
+	    else
+		svn up model-collect
+	    fi
+	done
+    fi
+else
+    svn co http://svn.greenstone.org/other-projects/nightly-tasks/diffcol/trunk/model-collect
+fi
+
+# Make a copy of the model-collect named as the new collect 
+# (or if collections are specified in the cmdline arguments, copy just these over from model-collect into collect)
+# Then remove the copy's .svn folders
 if [ -e collect_orig ]; then
-    if [ -e collect ]; then
-	rm -rf collect
-    fi
-    cp -r model-collect collect
-fi
-
-#cd collect
-#find . -name ".svn" -type d -exec rm -rf {} \;
-#cd ..
-find collect -name ".svn" -type d -exec rm -rf {} \; #2>&1 > /dev/null
-
-
-# Now delete all empty dirs in the copy of model-collect (since the dirs will not have .svn folders in them anymore)
-# http://www.commandlinefu.com/commands/view/5131/recursively-remove-all-empty-directories
-#find collect -type d -empty -delete
+    if [ ! -e collect ]; then
+	cp -r model-collect collect	
+	find collect -name ".svn" -type d -exec rm -rf {} \; #2>&1 > /dev/null
+    else
+	if [ "$1" == "" ]; then
+	    rm -rf collect
+	    cp -r model-collect collect
+	    find collect -name ".svn" -type d -exec rm -rf {} \;
+	else
+	    for collection in "$@"; do
+		if [ -e collect/$collection ]; then
+		    rm -rf collect/$collection
+		fi
+		cp -r model-collect/$collection collect/$collection
+		find collect/$collection -name ".svn" -type d -exec rm -rf {} \;
+	    done
+	fi
+    fi
+fi
 
 # Set up the Greenstone environment for building
@@ -353,16 +417,24 @@
 
 if [ "$1" == "" ]; then
-	# all_collections	
+
+    # all_collections	
     #for each collection, import, build, move building to index
     for collection in collect/*; do
-	single_collection $collection;
+	build_single_collection $collection;
+
 	if [ "x$mode" != "x" ]; then
-	    # Now delete all empty dirs in the copy of model-collect (since the dirs will not have .svn folders in them anymore)
-            # http://www.commandlinefu.com/commands/view/5131/recursively-remove-all-empty-directories
-#	    find collect/$collection/archives/HASH* -type d -empty -delete
-#	    find collect/$collection/index/assoc/HASH* -type d -empty -delete
-	    update_single_collection $collection $mode;
+	    svn_process_single_collection $collection
+
+	    if [ "x$mode" == "xsvnupdate" ]; then
+		update_single_collection $collection
+	    fi
 	fi
     done
+
+    # having rebuilt all the collections, just the processing for svndelete/update remains:
+    if [ "x$mode" == "xsvndelete" ]; then
+	svn_delete
+    fi
+
 else
     # Command-line args are a list of collections, 
@@ -372,9 +444,12 @@
 	collection=collect/$collection
 	if test -e $collection; then   
-	    single_collection $collection;
+	    build_single_collection $collection;
+
 	    if [ "x$mode" != "x" ]; then
-#		find collect/$collection/archives/HASH* -type d -empty -delete
-#		find collect/$collection/index/assoc/HASH* -type d -empty -delete
-		update_single_collection $collection $mode;
+		svn_process_single_collection $collection
+
+		if [ "x$mode" != "x" ]; then
+		    update_single_collection $collection
+		fi
 	    fi
 	else
@@ -382,4 +457,10 @@
 	fi
     done
+
+    # having rebuilt the specified collections above, just the processing for svndelete/update remains
+    if [ "x$mode" == "xsvndelete" ]; then
+	svn_delete $@
+    fi
+
 fi
 
@@ -405,6 +486,15 @@
 
 
-# The following in a separate script file will delete all folders from model-collect that are
+# deletes empty dirs
+#	    find collect/$collection/archives/HASH* -type d -empty -delete
+#	    find collect/$collection/index/assoc/HASH* -type d -empty -delete
+
+# To recursively delete all empty dirs in the copy of model-collect (since the dirs will not have .svn folders in them anymore)
+# http://www.commandlinefu.com/commands/view/5131/recursively-remove-all-empty-directories
+#find collect -type d -empty -delete
+
+# The following when put in a separate script file will delete all folders from model-collect that are
 # empty in the copied collection (all folders which contain only a .svn subfolder in model-collect)
+#!/bin/bash
 
 #for collection in collect/*; do
