source: other-projects/nightly-tasks/crons-and-scripts/do-snapshots-and-latest.sh@ 30002

Last change on this file since 30002 was 30002, checked in by ak19, 6 years ago

Further corrections to the nightly script generating the contents of the latest folder. It's hard to test it, as correct behaviour can only be confirmed overnight after the caveats have been generated.

  • Property svn:executable set to *
File size: 9.7 KB
Line 
1#!/bin/bash
2
3# This script is called each time a nightly GS file (binary or otherwise) gets uploaded to puka
4# to the caveat-emptor folder. This script is called from
5# nzdl@puka:[522]~/.ssh$ less authorized_keys
6
7
8######### GLOBAL VARS #########
9
10# http://stackoverflow.com/questions/1401482/yyyy-mm-dd-format-date-in-shell-script
11# we want . as separator to produce: yyyy.mm.dd
12
13today=`date +%Y.%m.%d`
14
15# http://askubuntu.com/questions/367136/how-do-i-read-a-variable-from-a-file
16
17gs2version=`cat /greenstone/greenstone.org/base/next-release.txt`
18gs3version=`cat /greenstone/greenstone.org/base/next-release-greenstone3.txt`
19
20# array of string literals of source component/distribution suffixes
21
22source_suffixes=("source-component.tar.gz" "source-component.zip" "source-distribution.tar.gz" "source-distribution.zip")
23
24# static array of string literals representing the caveat file suffixes we're interested in
25# see also 'Adding New Elements to the Original Array'
26# at http://www.yourownlinux.com/2013/10/working-with-arrays-in-bash-scripting.html
27
28suffixes=("linux" "linux-x64" "Lion-MacOS-intel.dmg" "MacOS-intel.dmg" "windows.exe" ${source_suffixes[@]})
29
30gs2_bin_prefix=Greenstone-$gs2version-candidate-$today-
31gs3_bin_prefix=Greenstone-$gs3version-candidate-$today-
32
33#echo "Date is $today"
34#echo "gs2ver is $gs2version"
35#echo "gs3ver is $gs3version"
36#echo "gs2_bin_prefix is $gs2_bin_prefix"
37#echo "gs3_bin_prefix is $gs3_bin_prefix"
38
39caveat_dir=/greenstone/greenstone.org/base/caveat-emptor/
40snapshots_dir=/greenstone/greenstone.org/base/release-snapshots/
41latest_dir=${caveat_dir}latest/
42
43###############################
44
45# Function that checks whether all of today's binaries for a gs_version (GS2 or GS3) are
46# present in the caveat folder
47# If so, it copies those bins over into the release-snapshots folder under new names
48
49# Non-ambiguous string concatenation: ${varname}string-literal
50# stackoverflow.com/questions/4181703/how-can-i-concatenate-string-variables-in-bash
51
52function snapshots_for_gs_version () {
53
54 # Local vs global variables in functions
55 # http://www.tldp.org/LDP/abs/html/localvar.html
56
57 local gs_version=$1
58
59 #check if we've found all gs_version nightly binaries
60 local foundallbins=true
61
62 for suffix in ${suffixes[@]}; do
63 local file=$caveat_dir
64 if [ "$gs_version" == "2" ]; then
65 file=$file$gs2_bin_prefix$suffix
66 else
67 file=$file$gs3_bin_prefix$suffix
68 fi
69
70 if [ ! -f $file ]; then
71 echo "File does not exist: $file"
72 foundallbins=false
73 #else
74 #echo "File exists: $file"
75 fi
76 done
77
78 echo ""
79
80 if [ "$foundallbins" == "true" ]; then
81 echo "All gs$gs_version binaries for $today generated"
82 echo ""
83
84 # delete the old stables
85 echo "Deleting ${snapshots_dir}Greenstone-${gs_version}s*"
86 rm ${snapshots_dir}Greenstone-${gs_version}s*
87 echo ""
88
89 # copy the gs2 binaries to release-snapshots folder
90 # under the new name Greenstone-${gs_version}sDATE-suffix, where gs_version = 2|3
91 # Use hardlinking instead of copy to save space
92
93 for suffix in ${suffixes[@]}; do
94 local file=$caveat_dir
95 if [ "$gs_version" == "2" ]; then
96 file=$file$gs2_bin_prefix$suffix
97 else
98 file=$file$gs3_bin_prefix$suffix
99 fi
100 local newfile=${snapshots_dir}Greenstone-${gs_version}s${today}-$suffix
101
102 # http://askubuntu.com/questions/108771/what-is-the-difference-between-a-hard-link-and-a-symbolic-link
103 # Q & A: The difference between hard and soft links at http://linuxgazette.tuwien.ac.at/105/pitcher.html
104 echo "Hardlinking file $file as $newfile"
105 ln $file $newfile
106
107 done
108
109 else
110 echo "Not all gs${gs_version} binaries for $today have been generated. (foundallbins: $foundallbins)"
111 fi
112
113 echo ""
114
115
116 # cdrom components only for GS2
117
118 if [ "$gs_version" == "2" ]; then
119
120 echo "GS2 CD ROM components"
121 echo ""
122
123 cd_suffixes=("linux" "linux-x64" "Lion-mac" "mac" "full-windows" "windows")
124 cdfile_src=cdrom-components-${gs2version}-candidate-${today}-
125
126 local foundallcds=true
127
128 for suffix in ${cd_suffixes[@]}; do
129
130 local file=${caveat_dir}$cdfile_src${suffix}.tar.gz
131
132 if [ ! -f $file ]; then
133 foundallcds=false
134 echo "Could not find cdrom component $file"
135 fi
136 done
137
138 # if all cdrom components present, then copy across under new names
139 if [ "$foundallcds" == "true" ]; then
140
141 # delete the old cdrom components from release-snapshots
142 echo "Deleting ${snapshots_dir}cdrom-components-2s*"
143 rm ${snapshots_dir}cdrom-components-2s*
144 echo ""
145
146 # copy across the new cdrom components
147 # use hardlinking instead of copying to save space
148
149 for suffix in ${cd_suffixes[@]}; do
150
151 local srcfile=${caveat_dir}$cdfile_src${suffix}.tar.gz
152 local destfile=${snapshots_dir}cdrom-components-2s${today}-${suffix}.tar.gz
153 echo "Hardlinking $srcfile as $destfile"
154 ln $srcfile $destfile
155 done
156 fi
157
158 echo ""
159 fi
160
161
162}
163
164# Inspect the most recently added file in the caveat folder and iff it's a GS binary or src file,
165# create a hardlink for it under the appropriate name in the 'latest' subfolder
166# and delete the earlier variant of the same in 'latest'.
167# In time may have to start including the GS documented examples archive files generated overnight.
168#
169# General: http://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash
170
171function make_latest () {
172
173 echo ""
174 #echo "In MAKE_LATEST"
175
176 # Using local variable to loop through files, seen in http://mywiki.wooledge.org/ParsingLs
177 local latest
178 local file
179 local suffix
180
181 # find the latest file in caveat, since this script is called upon each upload to puka's caveat
182 # http://stackoverflow.com/questions/5885934/bash-function-to-find-newest-file-matching-pattern
183 # http://mywiki.wooledge.org/BashFAQ/003
184 # http://tldp.org/LDP/abs/html/fto.html, where f1 -nt f2 means file f1 is newer than file f2
185 # http://superuser.com/questions/687523/get-the-last-file-name-in-a-variable-in-bash
186
187 unset -v latest
188 #for file in "${caveat_dir}"*; do
189 #then file would be the full path. Instead, cd into the caveat dir to just get the filename
190 cd $caveat_dir
191 for file in *; do
192 [[ -f $file ]] && [[ "$file" == "Greenstone-"* ]] && [[ "$file" != *"Greenstone3-latest-source.tar.gz" ]] && [[ $file -nt $latest ]] && latest=$file
193 done
194 cd -
195
196 # latest will not be undef, because this script is only ever called upon a legitimate
197 # file upload into puka's caveat-emptor location
198 # E.g. for testing, latest=Greenstone-3.07-candidate-2015.06.30-linux-x64
199 echo "Latest file in caveat is: $latest"
200
201 # splits the filename $latest by hyphens, -, and then gets the 2nd substring
202 # in this case the GS2 or GS3 full version number
203 # http://stackoverflow.com/questions/428109/extract-substring-in-bash
204 local matchfile=`echo $latest| cut -d'-' -f 2`
205 echo "Found gs version prefix: $matchfile"
206
207 # Iff this is a GS binary/src file that we want to maintain in the caveat/latest folder,
208 # then remove the older version of the same file from caveat/latest folder, if any,
209 # and hardlink the latest file into caveat/latest
210
211 # Check whether the latest file ends with the binary/src suffix
212 # https://viewsby.wordpress.com/2013/09/06/bash-string-ends-with/
213 # single or double square brackets for testing conditions, [] vs [[]]
214 # http://stackoverflow.com/questions/669452/is-preferable-over-in-bash-scripts
215 # Need double brackets below, [[]], for string expansion with wildcard
216
217
218 # if the latest file is a src component or src dist, then all src comps and dists
219 # would have been uploaded simultaneously for that GS version. Check they're all there
220 # and remove any older equivalents in the caveat/latest directory. Then copy all the
221 # *source* files for this GS version over to caveat/latest
222
223 echo ""
224 if [[ "$latest" == *"-source-"* ]]; then
225 for suffix in ${source_suffixes[@]}; do
226 local srcfile=Greenstone-${matchfile}-candidate-${today}-${suffix}
227 local regexfile=Greenstone-${matchfile}-candidate-*-${suffix}
228 if [[ -f ${caveat_dir}$srcfile ]]; then
229 echo "Removing ${latest_dir}$regexfile"
230 echo "and replacing with ${latest_dir}$srcfile"
231 rm ${latest_dir}$regexfile
232 ln ${caveat_dir}$srcfile ${latest_dir}$srcfile
233 fi
234 done
235
236 else
237 # not src component or dist, but binary file, which are uploaded one at a time.
238
239 for suffix in ${suffixes[@]}; do
240 if [[ "$latest" == *"$suffix" ]]; then
241 echo "Found that suffix matched: $suffix"
242
243 matchfile=${latest_dir}Greenstone-${matchfile}-candidate-*-${suffix}
244
245 # remove the matching file from the caveat/latest folder
246 # we don't know it's date, so we substituted that portion with a * as wildcard
247 # E.g. matchfile=<caveat>/latest/Greenstone-2.87-candidate-*-source-component.tar.gz
248
249 echo "Removing $matchfile"
250 rm $matchfile
251
252 # make a hardlink of the $latest file into the caveat_latest folder
253 echo "Making a hardlink to ${caveat_dir}$latest in ${latest_dir}$latest"
254 ln ${caveat_dir}$latest ${latest_dir}$latest
255 #else
256 #echo "$latest did not match $suffix"
257 fi
258 done
259 fi
260
261 echo ""
262}
263
264####################
265
266# the 'main'
267
268# This script is called upon each upload to puka's caveat-emptor by
269# the file nzdl@puka:[522]~/.ssh/authorized_keys
270
271# Check if conditions are true to create snapshots out of today's GS2 bins, these conditions
272# being that all src and bins of GS2 have been generated for today.
273# And if so, hardlink these src and bin files into the release-snapshots dir.
274# Repeat for GS3.
275# Finally, for each uploaded file that is a Greenstone src or binary file in caveat
276# hardlink it into the caveat/latest folder to keep a copy of the most-recent version of that
277# bin/file that was successfully generated.
278
279snapshots_for_gs_version "2"
280snapshots_for_gs_version "3"
281make_latest
Note: See TracBrowser for help on using the repository browser.