root/other-projects/nightly-tasks/crons-and-scripts/do-snapshots-and-latest.sh @ 30004

Revision 30004, 10.3 KB (checked in by ak19, 5 years ago)

Adding DEC to caveat latest too, although nightly DEC have never been checked for correctness, to see that they behave the way they did when Oran was here.

  • Property svn:executable set to *
Line 
1#!/bin/bash
2
3# This script is called each time a nightly GS file (binary or otherwise) gets uploaded to puka
4# to the caveat-emptor folder. This script is called from
5# nzdl@puka:[522]~/.ssh$ less authorized_keys
6
7
8######### GLOBAL VARS #########
9
10# http://stackoverflow.com/questions/1401482/yyyy-mm-dd-format-date-in-shell-script
11# we want . as separator to produce: yyyy.mm.dd
12
13today=`date +%Y.%m.%d`
14
15# http://askubuntu.com/questions/367136/how-do-i-read-a-variable-from-a-file
16
17gs2version=`cat /greenstone/greenstone.org/base/next-release.txt`
18gs3version=`cat /greenstone/greenstone.org/base/next-release-greenstone3.txt`
19
20# array of string literals of source component/distribution suffixes
21
22source_suffixes=("source-component.tar.gz" "source-component.zip" "source-distribution.tar.gz" "source-distribution.zip")
23
24# static array of string literals representing the caveat file suffixes we're interested in
25# see also 'Adding New Elements to the Original Array'
26# at http://www.yourownlinux.com/2013/10/working-with-arrays-in-bash-scripting.html
27
28suffixes=("linux" "linux-x64" "Lion-MacOS-intel.dmg" "MacOS-intel.dmg" "windows.exe" ${source_suffixes[@]})
29
30gs2_bin_prefix=Greenstone-$gs2version-candidate-$today-
31gs3_bin_prefix=Greenstone-$gs3version-candidate-$today-
32
33#echo "Date is $today"
34#echo "gs2ver is $gs2version"
35#echo "gs3ver is $gs3version"
36#echo "gs2_bin_prefix is $gs2_bin_prefix"
37#echo "gs3_bin_prefix is $gs3_bin_prefix"
38
39caveat_dir=/greenstone/greenstone.org/base/caveat-emptor/
40snapshots_dir=/greenstone/greenstone.org/base/release-snapshots/
41latest_dir=${caveat_dir}latest/
42
43###############################
44
45# Function that checks whether all of today's binaries for a gs_version (GS2 or GS3) are
46# present in the caveat folder
47# If so, it copies those bins over into the release-snapshots folder under new names
48
49# Non-ambiguous string concatenation: ${varname}string-literal
50# stackoverflow.com/questions/4181703/how-can-i-concatenate-string-variables-in-bash
51
52function snapshots_for_gs_version () {
53
54    # Local vs global variables in functions
55    # http://www.tldp.org/LDP/abs/html/localvar.html
56
57    local gs_version=$1
58
59    #check if we've found all gs_version nightly binaries
60    local foundallbins=true
61   
62    for suffix in ${suffixes[@]}; do
63    local file=$caveat_dir
64    if [ "$gs_version" == "2" ]; then
65        file=$file$gs2_bin_prefix$suffix
66    else
67        file=$file$gs3_bin_prefix$suffix
68    fi
69
70    if [ ! -f $file ]; then
71        echo "File does not exist: $file"
72        foundallbins=false
73    #else
74        #echo "File exists: $file"
75    fi
76    done
77
78    echo ""
79
80    if [ "$foundallbins" == "true" ]; then
81    echo "All gs$gs_version binaries for $today generated"
82    echo ""
83
84    # delete the old stables
85    echo "Deleting ${snapshots_dir}Greenstone-${gs_version}s*"
86    rm ${snapshots_dir}Greenstone-${gs_version}s*
87    echo ""
88   
89    # copy the gs2 binaries to release-snapshots folder
90    # under the new name Greenstone-${gs_version}sDATE-suffix, where gs_version = 2|3
91    # Use hardlinking instead of copy to save space
92
93    for suffix in ${suffixes[@]}; do   
94        local file=$caveat_dir
95        if [ "$gs_version" == "2" ]; then
96        file=$file$gs2_bin_prefix$suffix
97        else
98        file=$file$gs3_bin_prefix$suffix
99        fi
100        local newfile=${snapshots_dir}Greenstone-${gs_version}s${today}-$suffix
101       
102        # http://askubuntu.com/questions/108771/what-is-the-difference-between-a-hard-link-and-a-symbolic-link
103        # Q & A: The difference between hard and soft links at http://linuxgazette.tuwien.ac.at/105/pitcher.html
104        echo "Hardlinking file $file as $newfile"
105        ln $file $newfile
106
107    done
108   
109    else
110    echo "Not all gs${gs_version} binaries for $today have been generated. (foundallbins: $foundallbins)"
111    fi
112
113    echo ""
114
115
116    # cdrom components only for GS2
117
118    if [ "$gs_version" == "2" ]; then   
119
120    echo "GS2 CD ROM components"
121    echo ""
122
123    cd_suffixes=("linux" "linux-x64" "Lion-mac" "mac" "full-windows" "windows")
124    cdfile_src=cdrom-components-${gs2version}-candidate-${today}-
125
126    local foundallcds=true
127
128    for suffix in ${cd_suffixes[@]}; do
129
130        local file=${caveat_dir}$cdfile_src${suffix}.tar.gz
131
132        if [ ! -f $file ]; then
133        foundallcds=false
134        echo "Could not find cdrom component $file"
135        fi
136    done
137   
138    # if all cdrom components present, then copy across under new names
139    if [ "$foundallcds" == "true" ]; then
140
141        # delete the old cdrom components from release-snapshots
142        echo "Deleting ${snapshots_dir}cdrom-components-2s*"
143        rm ${snapshots_dir}cdrom-components-2s*
144        echo ""
145       
146        # copy across the new cdrom components
147        # use hardlinking instead of copying to save space
148
149        for suffix in ${cd_suffixes[@]}; do
150
151        local srcfile=${caveat_dir}$cdfile_src${suffix}.tar.gz
152        local destfile=${snapshots_dir}cdrom-components-2s${today}-${suffix}.tar.gz
153        echo "Hardlinking $srcfile as $destfile"
154        ln $srcfile $destfile
155        done
156    fi
157
158    echo ""
159    fi
160   
161
162}
163
164# Inspect the most recently added file in the caveat folder and iff it's a GS binary or src file,
165# create a hardlink for it under the appropriate name in the 'latest' subfolder
166# and delete the earlier variant of the same in 'latest'.
167# In time may have to start including the GS documented examples archive files generated overnight.
168#
169# General: http://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash
170
171function make_latest () {
172   
173    echo ""
174    #echo "In MAKE_LATEST"
175
176    # Using local variable to loop through files, seen in http://mywiki.wooledge.org/ParsingLs
177    local latest
178    local file
179    local suffix
180   
181    # find the latest file in caveat, since this script is called upon each upload to puka's caveat
182    # http://stackoverflow.com/questions/5885934/bash-function-to-find-newest-file-matching-pattern
183    # http://mywiki.wooledge.org/BashFAQ/003
184    # http://tldp.org/LDP/abs/html/fto.html, where f1 -nt f2 means file f1 is newer than file f2
185    # http://superuser.com/questions/687523/get-the-last-file-name-in-a-variable-in-bash
186
187    unset -v latest
188    #for file in "${caveat_dir}"*; do
189    #then file would be the full path. Instead, cd into the caveat dir to just get the filename
190    cd $caveat_dir
191    for file in *; do
192    [[ -f $file ]] && [[ "$file" == "Greenstone-"* ]] && [[ "$file" != *"Greenstone3-latest-source.tar.gz" ]] && [[ $file -nt $latest ]] && latest=$file
193    done
194    cd -
195
196    # latest will not be undef, because this script is only ever called upon a legitimate
197    # file upload into puka's caveat-emptor location   
198    # E.g. for testing, latest=Greenstone-3.07-candidate-2015.06.30-linux-x64
199
200    echo "Latest file in caveat is: $latest"
201
202    # splits the filename $latest by hyphens, -, and then gets the 2nd substring
203    # in this case the GS2 or GS3 full version number
204    # http://stackoverflow.com/questions/428109/extract-substring-in-bash
205
206    local matchfile=`echo $latest| cut -d'-' -f 2`
207    echo "Found gs version prefix: $matchfile"
208
209    # Iff this is a GS binary/src file that we want to maintain in the caveat/latest folder,
210    # then remove the older version of the same file from caveat/latest folder, if any,
211    # and hardlink the latest file into caveat/latest
212
213    # Check whether the latest file ends with the binary/src suffix
214    # https://viewsby.wordpress.com/2013/09/06/bash-string-ends-with/
215    # single or double square brackets for testing conditions, [] vs [[]]
216    # http://stackoverflow.com/questions/669452/is-preferable-over-in-bash-scripts
217    # Need double brackets below, [[]], for string expansion with wildcard
218
219
220    # if the latest file is a src component or src dist, then all src comps and dists
221    # would have been uploaded simultaneously for that GS version. Check they're all there
222    # and remove any older equivalents in the caveat/latest directory. Then copy all the
223    # *source* files for this GS version over to caveat/latest
224
225    echo ""
226    if [[ "$latest" == *"-source-"* ]]; then
227    for suffix in ${source_suffixes[@]}; do
228        local srcfile=Greenstone-${matchfile}-candidate-${today}-${suffix}
229        local regexfile=Greenstone-${matchfile}-candidate-*-${suffix}
230        if [[ -f ${caveat_dir}$srcfile ]]; then
231        echo "Removing ${latest_dir}$regexfile"
232        echo "and replacing with ${latest_dir}$srcfile"
233        rm ${latest_dir}$regexfile
234        ln ${caveat_dir}$srcfile ${latest_dir}$srcfile
235        fi
236    done
237
238    elif [[ "$latest" == *"documented-examples-"* ]]; then
239    echo "DEC"
240
241    # both documented examples zip and tar.gz files would have been generated and uploaded together
242   
243    local file=Greenstone-documented-examples-${today}
244    local re_file=Greenstone-documented-examples-*
245    local dec_exts=(".tar.gz" ".zip")
246
247    for ext in ${dec_exts[@]}; do
248        if [[ -f ${caveat_dir}${file}.tar.gz ]]; then
249
250        echo "Removing ${latest_dir}${re_file}${ext}"
251        echo "and replacing with ${latest_dir}${file}${ext}"
252
253        rm ${latest_dir}${re_file}${ext}
254        ln ${caveat_dir}${file}${ext} ${latest_dir}${file}${ext}
255        fi
256    done
257   
258    else
259    # not src component or dist, but binary file, which are uploaded one at a time.
260
261    for suffix in ${suffixes[@]}; do
262        if [[ "$latest" == *"$suffix" ]]; then
263        echo "Found that suffix matched: $suffix"
264       
265        matchfile=${latest_dir}Greenstone-${matchfile}-candidate-*-${suffix}
266       
267            # remove the matching file from the caveat/latest folder
268        # we don't know it's date, so we substituted that portion with a * as wildcard
269        # E.g. matchfile=<caveat>/latest/Greenstone-2.87-candidate-*-source-component.tar.gz
270       
271        echo "Removing $matchfile"
272        rm $matchfile
273       
274        # make a hardlink of the $latest file into the caveat_latest folder
275        echo "Making a hardlink to ${caveat_dir}$latest in ${latest_dir}$latest"
276        ln ${caveat_dir}$latest ${latest_dir}$latest
277        #else
278        #echo "$latest did not match $suffix"
279        fi
280    done
281    fi
282
283    echo ""
284}
285
286####################
287
288# the 'main'
289
290# This script is called upon each upload to puka's caveat-emptor by
291# the file nzdl@puka:[522]~/.ssh/authorized_keys
292
293# Check if conditions are true to create snapshots out of today's GS2 bins, these conditions
294# being that all src and bins of GS2 have been generated for today.
295# And if so, hardlink these src and bin files into the release-snapshots dir.
296# Repeat for GS3.
297# Finally, for each uploaded file that is a Greenstone src or binary file in caveat
298# hardlink it into the caveat/latest folder to keep a copy of the most-recent version of that
299# bin/file that was successfully generated.
300
301snapshots_for_gs_version "2"
302snapshots_for_gs_version "3"
303make_latest
Note: See TracBrowser for help on using the browser.