root/other-projects/nightly-tasks/crons-and-scripts/do-snapshots-and-latest.sh @ 32022

Revision 32022, 10.5 KB (checked in by ak19, 2 years ago)

Fix intended to get stables uploaded to the Snapshots page again. Leopards have long ceased to get generated (and I no longer even have access to our Mac Leopard machine, which may be connected), so snapshots weren't going through, because they either all go through or none go through. The tentative fix is to remove Mac Leopard from the list of expected binaries/products. That seems to have fixed the issue when I ran the script manually, hopefully the fix will also work when the nightly binaries go through overnight. Need to check tomorrow.

  • Property svn:executable set to *
Line 
1#!/bin/bash
2
3# This script is called each time a nightly GS file (binary or otherwise) gets uploaded to puka
4# to the caveat-emptor folder. This script is called from
5# nzdl@puka:[522]~/.ssh$ less authorized_keys
6
7
8######### GLOBAL VARS #########
9
10# http://stackoverflow.com/questions/1401482/yyyy-mm-dd-format-date-in-shell-script
11# we want . as separator to produce: yyyy.mm.dd
12
13today=`date +%Y.%m.%d`
14
15# http://askubuntu.com/questions/367136/how-do-i-read-a-variable-from-a-file
16
17gs2version=`cat /greenstone/greenstone.org/base/next-release.txt`
18gs3version=`cat /greenstone/greenstone.org/base/next-release-greenstone3.txt`
19
20# array of string literals of source component/distribution suffixes
21
22source_suffixes=("source-component.tar.gz" "source-component.zip" "source-distribution.tar.gz" "source-distribution.zip")
23
24# static array of string literals representing the caveat file suffixes we're interested in
25# see also 'Adding New Elements to the Original Array'
26# at http://www.yourownlinux.com/2013/10/working-with-arrays-in-bash-scripting.html
27
28# Mac Leopard binaries are no longer automatically generated
29# (as I can't even log into our Mac Leopard machine)
30#suffixes=("linux" "linux-x64" "Lion-MacOS-intel.dmg" "MacOS-intel.dmg" "windows.exe" ${source_suffixes[@]})
31suffixes=("linux" "linux-x64" "Lion-MacOS-intel.dmg" "windows.exe" ${source_suffixes[@]})
32
33gs2_bin_prefix=Greenstone-$gs2version-candidate-$today-
34gs3_bin_prefix=Greenstone-$gs3version-candidate-$today-
35
36#echo "Date is $today"
37#echo "gs2ver is $gs2version"
38#echo "gs3ver is $gs3version"
39#echo "gs2_bin_prefix is $gs2_bin_prefix"
40#echo "gs3_bin_prefix is $gs3_bin_prefix"
41
42caveat_dir=/greenstone/greenstone.org/base/caveat-emptor/
43snapshots_dir=/greenstone/greenstone.org/base/release-snapshots/
44latest_dir=${caveat_dir}latest/
45
46###############################
47
48# Function that checks whether all of today's binaries for a gs_version (GS2 or GS3) are
49# present in the caveat folder
50# If so, it copies those bins over into the release-snapshots folder under new names
51
52# Non-ambiguous string concatenation: ${varname}string-literal
53# stackoverflow.com/questions/4181703/how-can-i-concatenate-string-variables-in-bash
54
55function snapshots_for_gs_version () {
56
57    # Local vs global variables in functions
58    # http://www.tldp.org/LDP/abs/html/localvar.html
59
60    local gs_version=$1
61
62    #check if we've found all gs_version nightly binaries
63    local foundallbins=true
64   
65    for suffix in ${suffixes[@]}; do
66    local file=$caveat_dir
67    if [ "$gs_version" == "2" ]; then
68        file=$file$gs2_bin_prefix$suffix
69    else
70        file=$file$gs3_bin_prefix$suffix
71    fi
72
73    if [ ! -f $file ]; then
74        echo "File does not exist: $file"
75        foundallbins=false
76    #else
77        #echo "File exists: $file"
78    fi
79    done
80
81    echo ""
82
83    if [ "$foundallbins" == "true" ]; then
84    echo "All gs$gs_version binaries for $today generated"
85    echo ""
86
87    # delete the old stables
88    echo "Deleting ${snapshots_dir}Greenstone-${gs_version}s*"
89    rm ${snapshots_dir}Greenstone-${gs_version}s*
90    echo ""
91   
92    # copy the gs2 binaries to release-snapshots folder
93    # under the new name Greenstone-${gs_version}sDATE-suffix, where gs_version = 2|3
94    # Use hardlinking instead of copy to save space
95
96    for suffix in ${suffixes[@]}; do   
97        local file=$caveat_dir
98        if [ "$gs_version" == "2" ]; then
99        file=$file$gs2_bin_prefix$suffix
100        else
101        file=$file$gs3_bin_prefix$suffix
102        fi
103        local newfile=${snapshots_dir}Greenstone-${gs_version}s${today}-$suffix
104       
105        # http://askubuntu.com/questions/108771/what-is-the-difference-between-a-hard-link-and-a-symbolic-link
106        # Q & A: The difference between hard and soft links at http://linuxgazette.tuwien.ac.at/105/pitcher.html
107        echo "Hardlinking file $file as $newfile"
108        ln $file $newfile
109
110    done
111   
112    else
113    echo "Not all gs${gs_version} binaries for $today have been generated. (foundallbins: $foundallbins)"
114    fi
115
116    echo ""
117
118
119    # cdrom components only for GS2
120
121    if [ "$gs_version" == "2" ]; then   
122
123    echo "GS2 CD ROM components"
124    echo ""
125
126    cd_suffixes=("linux" "linux-x64" "Lion-mac" "mac" "full-windows" "windows")
127    cdfile_src=cdrom-components-${gs2version}-candidate-${today}-
128
129    local foundallcds=true
130
131    for suffix in ${cd_suffixes[@]}; do
132
133        local file=${caveat_dir}$cdfile_src${suffix}.tar.gz
134
135        if [ ! -f $file ]; then
136        foundallcds=false
137        echo "Could not find cdrom component $file"
138        fi
139    done
140   
141    # if all cdrom components present, then copy across under new names
142    if [ "$foundallcds" == "true" ]; then
143
144        # delete the old cdrom components from release-snapshots
145        echo "Deleting ${snapshots_dir}cdrom-components-2s*"
146        rm ${snapshots_dir}cdrom-components-2s*
147        echo ""
148       
149        # copy across the new cdrom components
150        # use hardlinking instead of copying to save space
151
152        for suffix in ${cd_suffixes[@]}; do
153
154        local srcfile=${caveat_dir}$cdfile_src${suffix}.tar.gz
155        local destfile=${snapshots_dir}cdrom-components-2s${today}-${suffix}.tar.gz
156        echo "Hardlinking $srcfile as $destfile"
157        ln $srcfile $destfile
158        done
159    fi
160
161    echo ""
162    fi
163   
164
165}
166
167# Inspect the most recently added file in the caveat folder and iff it's a GS binary or src file,
168# create a hardlink for it under the appropriate name in the 'latest' subfolder
169# and delete the earlier variant of the same in 'latest'.
170# In time may have to start including the GS documented examples archive files generated overnight.
171#
172# General: http://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash
173
174function make_latest () {
175   
176    echo ""
177    #echo "In MAKE_LATEST"
178
179    # Using local variable to loop through files, seen in http://mywiki.wooledge.org/ParsingLs
180    local latest
181    local file
182    local suffix
183   
184    # find the latest file in caveat, since this script is called upon each upload to puka's caveat
185    # http://stackoverflow.com/questions/5885934/bash-function-to-find-newest-file-matching-pattern
186    # http://mywiki.wooledge.org/BashFAQ/003
187    # http://tldp.org/LDP/abs/html/fto.html, where f1 -nt f2 means file f1 is newer than file f2
188    # http://superuser.com/questions/687523/get-the-last-file-name-in-a-variable-in-bash
189
190    unset -v latest
191    #for file in "${caveat_dir}"*; do
192    #then file would be the full path. Instead, cd into the caveat dir to just get the filename
193    cd $caveat_dir
194    for file in *; do
195    [[ -f $file ]] && [[ "$file" == "Greenstone-"* ]] && [[ "$file" != *"Greenstone3-latest-source.tar.gz" ]] && [[ $file -nt $latest ]] && latest=$file
196    done
197    cd -
198
199    # latest will not be undef, because this script is only ever called upon a legitimate
200    # file upload into puka's caveat-emptor location   
201    # e.g. for testing, latest=Greenstone-3.07-candidate-2015.06.30-linux-x64
202
203    echo "Latest file in caveat is: $latest"
204
205    # splits the filename $latest by hyphens, -, and then gets the 2nd substring
206    # in this case the GS2 or GS3 full version number
207    # http://stackoverflow.com/questions/428109/extract-substring-in-bash
208
209    local matchfile=`echo $latest| cut -d'-' -f 2`
210    echo "Found gs version prefix: $matchfile"
211
212    # Iff this is a GS binary/src file that we want to maintain in the caveat/latest folder,
213    # then remove the older version of the same file from caveat/latest folder, if any,
214    # and hardlink the latest file into caveat/latest
215
216    # Check whether the latest file ends with the binary/src suffix
217    # https://viewsby.wordpress.com/2013/09/06/bash-string-ends-with/
218    # single or double square brackets for testing conditions, [] vs [[]]
219    # http://stackoverflow.com/questions/669452/is-preferable-over-in-bash-scripts
220    # Need double brackets below, [[]], for string expansion with wildcard
221
222
223    # if the latest file is a src component or src dist, then all src comps and dists
224    # would have been uploaded simultaneously for that GS version. Check they're all there
225    # and remove any older equivalents in the caveat/latest directory. Then copy all the
226    # *source* files for this GS version over to caveat/latest
227
228    echo ""
229    if [[ "$latest" == *"-source-"* ]]; then
230    for suffix in ${source_suffixes[@]}; do
231        local srcfile=Greenstone-${matchfile}-candidate-${today}-${suffix}
232        local regexfile=Greenstone-${matchfile}-candidate-*-${suffix}
233        if [[ -f ${caveat_dir}$srcfile ]]; then
234        echo "Removing ${latest_dir}$regexfile"
235        echo "and replacing with ${latest_dir}$srcfile"
236        rm ${latest_dir}$regexfile
237        ln ${caveat_dir}$srcfile ${latest_dir}$srcfile
238        fi
239    done
240
241    elif [[ "$latest" == *"documented-examples-"* ]]; then
242    echo "DEC"
243
244    # both documented examples zip and tar.gz files would have been generated and uploaded together
245   
246    local file=Greenstone-documented-examples-${today}
247    local re_file=Greenstone-documented-examples-*
248    local dec_exts=(".tar.gz" ".zip")
249
250    for ext in ${dec_exts[@]}; do
251        if [[ -f ${caveat_dir}${file}.tar.gz ]]; then
252
253        echo "Removing ${latest_dir}${re_file}${ext}"
254        echo "and replacing with ${latest_dir}${file}${ext}"
255
256        rm ${latest_dir}${re_file}${ext}
257        ln ${caveat_dir}${file}${ext} ${latest_dir}${file}${ext}
258        fi
259    done
260   
261    else
262    # not src component or dist, but binary file, which are uploaded one at a time.
263
264    for suffix in ${suffixes[@]}; do
265        if [[ "$latest" == *"$suffix" ]]; then
266        echo "Found that suffix matched: $suffix"
267       
268        matchfile=${latest_dir}Greenstone-${matchfile}-candidate-*-${suffix}
269       
270            # remove the matching file from the caveat/latest folder
271        # we don't know it's date, so we substituted that portion with a * as wildcard
272        # E.g. matchfile=<caveat>/latest/Greenstone-2.87-candidate-*-source-component.tar.gz
273       
274        echo "Removing $matchfile"
275        rm $matchfile
276       
277        # make a hardlink of the $latest file into the caveat_latest folder
278        echo "Making a hardlink to ${caveat_dir}$latest in ${latest_dir}$latest"
279        ln ${caveat_dir}$latest ${latest_dir}$latest
280        #else
281        #echo "$latest did not match $suffix"
282        fi
283    done
284    fi
285
286    echo ""
287}
288
289####################
290
291# the 'main'
292
293# This script is called upon each upload to puka's caveat-emptor by
294# the file nzdl@puka:[522]~/.ssh/authorized_keys
295
296# Check if conditions are true to create snapshots out of today's GS2 bins, these conditions
297# being that all src and bins of GS2 have been generated for today.
298# And if so, hardlink these src and bin files into the release-snapshots dir.
299# Repeat for GS3.
300# Finally, for each uploaded file that is a Greenstone src or binary file in caveat
301# hardlink it into the caveat/latest folder to keep a copy of the most-recent version of that
302# bin/file that was successfully generated.
303
304snapshots_for_gs_version "2"
305snapshots_for_gs_version "3"
306make_latest
Note: See TracBrowser for help on using the browser.