root/other-projects/nightly-tasks/crons-and-scripts/do-snapshots-and-latest.sh @ 30002

Revision 30002, 9.7 KB (checked in by ak19, 5 years ago)

Further corrections to the nightly script generating the contents of the latest folder. It's hard to test it, as correct behaviour can only be confirmed overnight after the caveats have been generated.

  • Property svn:executable set to *
Line 
1#!/bin/bash
2
3# This script is called each time a nightly GS file (binary or otherwise) gets uploaded to puka
4# to the caveat-emptor folder. This script is called from
5# nzdl@puka:[522]~/.ssh$ less authorized_keys
6
7
8######### GLOBAL VARS #########
9
10# http://stackoverflow.com/questions/1401482/yyyy-mm-dd-format-date-in-shell-script
11# we want . as separator to produce: yyyy.mm.dd
12
13today=`date +%Y.%m.%d`
14
15# http://askubuntu.com/questions/367136/how-do-i-read-a-variable-from-a-file
16
17gs2version=`cat /greenstone/greenstone.org/base/next-release.txt`
18gs3version=`cat /greenstone/greenstone.org/base/next-release-greenstone3.txt`
19
20# array of string literals of source component/distribution suffixes
21
22source_suffixes=("source-component.tar.gz" "source-component.zip" "source-distribution.tar.gz" "source-distribution.zip")
23
24# static array of string literals representing the caveat file suffixes we're interested in
25# see also 'Adding New Elements to the Original Array'
26# at http://www.yourownlinux.com/2013/10/working-with-arrays-in-bash-scripting.html
27
28suffixes=("linux" "linux-x64" "Lion-MacOS-intel.dmg" "MacOS-intel.dmg" "windows.exe" ${source_suffixes[@]})
29
30gs2_bin_prefix=Greenstone-$gs2version-candidate-$today-
31gs3_bin_prefix=Greenstone-$gs3version-candidate-$today-
32
33#echo "Date is $today"
34#echo "gs2ver is $gs2version"
35#echo "gs3ver is $gs3version"
36#echo "gs2_bin_prefix is $gs2_bin_prefix"
37#echo "gs3_bin_prefix is $gs3_bin_prefix"
38
39caveat_dir=/greenstone/greenstone.org/base/caveat-emptor/
40snapshots_dir=/greenstone/greenstone.org/base/release-snapshots/
41latest_dir=${caveat_dir}latest/
42
43###############################
44
45# Function that checks whether all of today's binaries for a gs_version (GS2 or GS3) are
46# present in the caveat folder
47# If so, it copies those bins over into the release-snapshots folder under new names
48
49# Non-ambiguous string concatenation: ${varname}string-literal
50# stackoverflow.com/questions/4181703/how-can-i-concatenate-string-variables-in-bash
51
52function snapshots_for_gs_version () {
53
54    # Local vs global variables in functions
55    # http://www.tldp.org/LDP/abs/html/localvar.html
56
57    local gs_version=$1
58
59    #check if we've found all gs_version nightly binaries
60    local foundallbins=true
61   
62    for suffix in ${suffixes[@]}; do
63    local file=$caveat_dir
64    if [ "$gs_version" == "2" ]; then
65        file=$file$gs2_bin_prefix$suffix
66    else
67        file=$file$gs3_bin_prefix$suffix
68    fi
69
70    if [ ! -f $file ]; then
71        echo "File does not exist: $file"
72        foundallbins=false
73    #else
74        #echo "File exists: $file"
75    fi
76    done
77
78    echo ""
79
80    if [ "$foundallbins" == "true" ]; then
81    echo "All gs$gs_version binaries for $today generated"
82    echo ""
83
84    # delete the old stables
85    echo "Deleting ${snapshots_dir}Greenstone-${gs_version}s*"
86    rm ${snapshots_dir}Greenstone-${gs_version}s*
87    echo ""
88   
89    # copy the gs2 binaries to release-snapshots folder
90    # under the new name Greenstone-${gs_version}sDATE-suffix, where gs_version = 2|3
91    # Use hardlinking instead of copy to save space
92
93    for suffix in ${suffixes[@]}; do   
94        local file=$caveat_dir
95        if [ "$gs_version" == "2" ]; then
96        file=$file$gs2_bin_prefix$suffix
97        else
98        file=$file$gs3_bin_prefix$suffix
99        fi
100        local newfile=${snapshots_dir}Greenstone-${gs_version}s${today}-$suffix
101       
102        # http://askubuntu.com/questions/108771/what-is-the-difference-between-a-hard-link-and-a-symbolic-link
103        # Q & A: The difference between hard and soft links at http://linuxgazette.tuwien.ac.at/105/pitcher.html
104        echo "Hardlinking file $file as $newfile"
105        ln $file $newfile
106
107    done
108   
109    else
110    echo "Not all gs${gs_version} binaries for $today have been generated. (foundallbins: $foundallbins)"
111    fi
112
113    echo ""
114
115
116    # cdrom components only for GS2
117
118    if [ "$gs_version" == "2" ]; then   
119
120    echo "GS2 CD ROM components"
121    echo ""
122
123    cd_suffixes=("linux" "linux-x64" "Lion-mac" "mac" "full-windows" "windows")
124    cdfile_src=cdrom-components-${gs2version}-candidate-${today}-
125
126    local foundallcds=true
127
128    for suffix in ${cd_suffixes[@]}; do
129
130        local file=${caveat_dir}$cdfile_src${suffix}.tar.gz
131
132        if [ ! -f $file ]; then
133        foundallcds=false
134        echo "Could not find cdrom component $file"
135        fi
136    done
137   
138    # if all cdrom components present, then copy across under new names
139    if [ "$foundallcds" == "true" ]; then
140
141        # delete the old cdrom components from release-snapshots
142        echo "Deleting ${snapshots_dir}cdrom-components-2s*"
143        rm ${snapshots_dir}cdrom-components-2s*
144        echo ""
145       
146        # copy across the new cdrom components
147        # use hardlinking instead of copying to save space
148
149        for suffix in ${cd_suffixes[@]}; do
150
151        local srcfile=${caveat_dir}$cdfile_src${suffix}.tar.gz
152        local destfile=${snapshots_dir}cdrom-components-2s${today}-${suffix}.tar.gz
153        echo "Hardlinking $srcfile as $destfile"
154        ln $srcfile $destfile
155        done
156    fi
157
158    echo ""
159    fi
160   
161
162}
163
164# Inspect the most recently added file in the caveat folder and iff it's a GS binary or src file,
165# create a hardlink for it under the appropriate name in the 'latest' subfolder
166# and delete the earlier variant of the same in 'latest'.
167# In time may have to start including the GS documented examples archive files generated overnight.
168#
169# General: http://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash
170
171function make_latest () {
172   
173    echo ""
174    #echo "In MAKE_LATEST"
175
176    # Using local variable to loop through files, seen in http://mywiki.wooledge.org/ParsingLs
177    local latest
178    local file
179    local suffix
180   
181    # find the latest file in caveat, since this script is called upon each upload to puka's caveat
182    # http://stackoverflow.com/questions/5885934/bash-function-to-find-newest-file-matching-pattern
183    # http://mywiki.wooledge.org/BashFAQ/003
184    # http://tldp.org/LDP/abs/html/fto.html, where f1 -nt f2 means file f1 is newer than file f2
185    # http://superuser.com/questions/687523/get-the-last-file-name-in-a-variable-in-bash
186
187    unset -v latest
188    #for file in "${caveat_dir}"*; do
189    #then file would be the full path. Instead, cd into the caveat dir to just get the filename
190    cd $caveat_dir
191    for file in *; do
192    [[ -f $file ]] && [[ "$file" == "Greenstone-"* ]] && [[ "$file" != *"Greenstone3-latest-source.tar.gz" ]] && [[ $file -nt $latest ]] && latest=$file
193    done
194    cd -
195
196    # latest will not be undef, because this script is only ever called upon a legitimate
197    # file upload into puka's caveat-emptor location   
198    # E.g. for testing, latest=Greenstone-3.07-candidate-2015.06.30-linux-x64
199    echo "Latest file in caveat is: $latest"
200
201    # splits the filename $latest by hyphens, -, and then gets the 2nd substring
202    # in this case the GS2 or GS3 full version number
203    # http://stackoverflow.com/questions/428109/extract-substring-in-bash
204    local matchfile=`echo $latest| cut -d'-' -f 2`
205    echo "Found gs version prefix: $matchfile"
206
207    # Iff this is a GS binary/src file that we want to maintain in the caveat/latest folder,
208    # then remove the older version of the same file from caveat/latest folder, if any,
209    # and hardlink the latest file into caveat/latest
210
211    # Check whether the latest file ends with the binary/src suffix
212    # https://viewsby.wordpress.com/2013/09/06/bash-string-ends-with/
213    # single or double square brackets for testing conditions, [] vs [[]]
214    # http://stackoverflow.com/questions/669452/is-preferable-over-in-bash-scripts
215    # Need double brackets below, [[]], for string expansion with wildcard
216
217
218    # if the latest file is a src component or src dist, then all src comps and dists
219    # would have been uploaded simultaneously for that GS version. Check they're all there
220    # and remove any older equivalents in the caveat/latest directory. Then copy all the
221    # *source* files for this GS version over to caveat/latest
222
223    echo ""
224    if [[ "$latest" == *"-source-"* ]]; then
225    for suffix in ${source_suffixes[@]}; do
226        local srcfile=Greenstone-${matchfile}-candidate-${today}-${suffix}
227        local regexfile=Greenstone-${matchfile}-candidate-*-${suffix}
228        if [[ -f ${caveat_dir}$srcfile ]]; then
229        echo "Removing ${latest_dir}$regexfile"
230        echo "and replacing with ${latest_dir}$srcfile"
231        rm ${latest_dir}$regexfile
232        ln ${caveat_dir}$srcfile ${latest_dir}$srcfile
233        fi
234    done
235   
236    else
237    # not src component or dist, but binary file, which are uploaded one at a time.
238
239    for suffix in ${suffixes[@]}; do
240        if [[ "$latest" == *"$suffix" ]]; then
241        echo "Found that suffix matched: $suffix"
242       
243        matchfile=${latest_dir}Greenstone-${matchfile}-candidate-*-${suffix}
244       
245            # remove the matching file from the caveat/latest folder
246        # we don't know it's date, so we substituted that portion with a * as wildcard
247        # E.g. matchfile=<caveat>/latest/Greenstone-2.87-candidate-*-source-component.tar.gz
248       
249        echo "Removing $matchfile"
250        rm $matchfile
251       
252        # make a hardlink of the $latest file into the caveat_latest folder
253        echo "Making a hardlink to ${caveat_dir}$latest in ${latest_dir}$latest"
254        ln ${caveat_dir}$latest ${latest_dir}$latest
255        #else
256        #echo "$latest did not match $suffix"
257        fi
258    done
259    fi
260
261    echo ""
262}
263
264####################
265
266# the 'main'
267
268# This script is called upon each upload to puka's caveat-emptor by
269# the file nzdl@puka:[522]~/.ssh/authorized_keys
270
271# Check if conditions are true to create snapshots out of today's GS2 bins, these conditions
272# being that all src and bins of GS2 have been generated for today.
273# And if so, hardlink these src and bin files into the release-snapshots dir.
274# Repeat for GS3.
275# Finally, for each uploaded file that is a Greenstone src or binary file in caveat
276# hardlink it into the caveat/latest folder to keep a copy of the most-recent version of that
277# bin/file that was successfully generated.
278
279snapshots_for_gs_version "2"
280snapshots_for_gs_version "3"
281make_latest
Note: See TracBrowser for help on using the browser.