Index: main/trunk/greenstone2/build-src/packages/Makefile
===================================================================
--- main/trunk/greenstone2/build-src/packages/Makefile	(revision 31828)
+++ main/trunk/greenstone2/build-src/packages/Makefile	(revision 31829)
@@ -29,4 +29,6 @@
 # is assumed that each package will have at least four rules: all, install,
 # clean, and distclean.
+# openssl is configured and compiled during the configure stage by
+# build-src/packages/configure. make clean and make distclean happen here.
 
 PACKAGEDIRS = 
@@ -40,4 +42,5 @@
 		wget/wget-1.15-gs cpan/XML-Parser-2.41 \
 		xlhtml/xlhtml-0.4.9.0 isis-gdl html-tidy/tidy
+OPENSSL_PKG = openssl/openssl-1.0.2l
 
 all:
@@ -54,5 +57,5 @@
 
 clean:
-	for subdir in $(PACKAGEDIRS); do \
+	for subdir in $(PACKAGEDIRS) $(OPENSSL_PKG); do \
           echo cleaning $@ in $$subdir; \
           (cd $$subdir && $(MAKE) clean || exit 0) ; \
@@ -60,6 +63,8 @@
 
 distclean:
-	for subdir in $(PACKAGEDIRS); do \
+	for subdir in $(PACKAGEDIRS) $(OPENSSL_PKG); do \
           echo cleaning $@ in $$subdir; \
           (cd $$subdir && $(MAKE) distclean || exit 0) ; \
         done
+	(cd openssl && rm -rf certs man misc private openssl.cnf || exit 0)
+	rm -rf $(OPENSSL_PKG)
Index: main/trunk/greenstone2/build-src/packages/Makefile.in
===================================================================
--- main/trunk/greenstone2/build-src/packages/Makefile.in	(revision 31828)
+++ main/trunk/greenstone2/build-src/packages/Makefile.in	(revision 31829)
@@ -29,4 +29,6 @@
 # is assumed that each package will have at least four rules: all, install,
 # clean, and distclean.
+# openssl is configured and compiled during the configure stage by
+# build-src/packages/configure. make clean and make distclean happen here.
 
 PACKAGEDIRS = 
@@ -40,4 +42,5 @@
 		wget/wget-1.15-gs cpan/XML-Parser-2.41 \
 		xlhtml/xlhtml-0.4.9.0 isis-gdl html-tidy/tidy
+OPENSSL_PKG = openssl/openssl-1.0.2l
 
 all:
@@ -54,5 +57,5 @@
 
 clean:
-	for subdir in $(PACKAGEDIRS); do \
+	for subdir in $(PACKAGEDIRS) $(OPENSSL_PKG); do \
           echo cleaning $@ in $$subdir; \
           (cd $$subdir && $(MAKE) clean || exit 0) ; \
@@ -60,6 +63,8 @@
 
 distclean:
-	for subdir in $(PACKAGEDIRS); do \
+	for subdir in $(PACKAGEDIRS) $(OPENSSL_PKG); do \
           echo cleaning $@ in $$subdir; \
           (cd $$subdir && $(MAKE) distclean || exit 0) ; \
         done
+	(cd openssl && rm -rf certs man misc private openssl.cnf || exit 0)
+	rm -rf $(OPENSSL_PKG)
Index: main/trunk/greenstone2/build-src/packages/configure
===================================================================
--- main/trunk/greenstone2/build-src/packages/configure	(revision 31828)
+++ main/trunk/greenstone2/build-src/packages/configure	(revision 31829)
@@ -150,4 +150,5 @@
 cd "$PACKAGES/openssl"
 openssl_pkgname=openssl-1.0.2l
+#openssl_pkgname=openssl-1.1.0f
 if test ! -d $openssl_pkgname ;
 then
@@ -158,4 +159,6 @@
 # configure
 # see the INSTALL file for OpenSSL to see config options for openssl
+# If you ever want to have the .a libraries that building openssl generates,
+# then use the --prefix and point it to the $bindir.
 # --prefix indicates where the generated lib, include and bin folders will go
 # --openssldir will contain the manuals and other non-essentials
@@ -248,9 +251,11 @@
 
 # configure wget
+wget_pkgname=wget-1.15-gs
+#wget_pkgname=wget-1.17.1-gs
 echo ""
 echo "Configuring wget"
 echo ""
 cd "$PACKAGES/wget"
-gzip -dc wget-1.15-gs.tar.gz | tar -xf -
+gzip -dc $wget_pkgname.tar.gz | tar -xf -
 if test x$crossOS = "xandroid" ;
 then
@@ -260,16 +265,27 @@
 fi
 
+# When compiling wget-1.15-gs without openssl as before:
 #./configure $CACHE_FILE --prefix=$PACKAGES/wget --without-ssl --bindir="$bindir" -disable-nls $HOSTTARGETFLAGS $wget_extraargs
 
-# We weren't compiling up wget statically before either
+# We weren't compiling up wget statically before either and we still aren't
 # However, to compile up wget (statically or not) with openssl, see
 # https://stackoverflow.com/questions/9817337/compiling-wget-with-static-linking-self-compiled-openssl-library-linking-issu
-# Note: if the cppflags and ldflags are set during the configure stage, as we now do,
-# then the make command needn't additionally set them as well. So just make, make install then works 
-echo cd "$PACKAGES/wget/wget-1.15-gs"
-echo CPPFLAGS="-I/$bindir/openssl/include" LDFLAGS="-L/$bindir/openssl/lib" ./configure $CACHE_FILE --prefix=$PACKAGES/wget --with-ssl=openssl --bindir="$bindir" -disable-nls $HOSTTARGETFLAGS $wget_extraargs
-cd "$PACKAGES/wget/wget-1.15-gs" && \
+# Note: contrary to the instructions at the link, if the cppflags and ldflags are set during the
+# configure stage, as we now do, then the make command needn't additionally set them as well.
+# So just make, make install works thereafter
+
+echo copying "$PACKAGES/wget/wgetrc-gs.in" file to "$bindir/wgetrc"
+cp "$PACKAGES/wget/wgetrc-gs.in" "$bindir/wgetrc"
+echo cd "$PACKAGES/wget/$wget_pkgname"
+
+# for wget-1.15-gs. (can also try adding --localstatedir="$bindir/wget" --enable-iri)
+echo CPPFLAGS="-I/$bindir/openssl/include" LDFLAGS="-L/$bindir/openssl/lib" ./configure $CACHE_FILE --prefix=$PACKAGES/wget --with-ssl=openssl --bindir="$bindir" -disable-nls $HOSTTARGETFLAGS $wget_extraarg
+cd "$PACKAGES/wget/$wget_pkgname" && \
 CPPFLAGS="-I/$bindir/openssl/include" LDFLAGS="-L/$bindir/openssl/lib" ./configure $CACHE_FILE --prefix=$PACKAGES/wget --with-ssl=openssl --bindir="$bindir" -disable-nls $HOSTTARGETFLAGS $wget_extraargs
 
+# for wget-1.17.1-gs. (can also try adding --localstatedir="$bindir/wget" --enable-iri)
+#echo OPENSSL_CFLAGS="-I/$bindir/openssl/include" OPENSSL_LIBS="-L/$bindir/openssl/lib -lssl -lcrypto" ./configure $CACHE_FILE --prefix=$PACKAGES/wget --with-ssl=openssl --with-openssl=auto --with-libssl-prefix="$bindir/openssl" --bindir="$bindir" -disable-nls $HOSTTARGETFLAGS $wget_extraargs
+#cd "$PACKAGES/wget/$wget_pkgname" && \
+#OPENSSL_CFLAGS="-I/$bindir/openssl/include" OPENSSL_LIBS="-L/$bindir/openssl/lib -lssl -lcrypto" ./configure $CACHE_FILE --prefix=$PACKAGES/wget --with-ssl=openssl --with-openssl=auto --with-libssl-prefix="$bindir/openssl" --bindir="$bindir"  -disable-nls $HOSTTARGETFLAGS $wget_extraargs
 
 ##
Index: main/trunk/greenstone2/build-src/packages/wget/README
===================================================================
--- main/trunk/greenstone2/build-src/packages/wget/README	(revision 31828)
+++ main/trunk/greenstone2/build-src/packages/wget/README	(revision 31829)
@@ -105,5 +105,5 @@
 
 
-9) We're now shifting to wget-1.17.1 which is installed on Ubuntu 16.04 and which work on https urls without the --no-check-certificate flag being necessary.
+9) We're now shifting to wget-1.17.1 which is installed on Ubuntu 16.04, and for which a windows binary compiled with OpenSSL is available. Both the linux system version and windows binary work on https urls without the --no-check-certificate flag being necessary. However, the compiled up Linux version still needs this flag, see under PROBLEM.
 
 This way our perl code can launch wget as before, without always passing that additional flag. Hopefully the output in the Download pane will be the same so that the donwload parsing will work.
@@ -118,4 +118,5 @@
 * configuring had initially failed, reporting that OPENSSL_CFLAGS and OPENSSL_LIBS need to be set if not wanting to use whatever pkg-config may find. To set LIBS variables, use one of these forms: LIBS="-L/path/to/lib" or LIBS="/path/to/lib/lib.a" or LIBS="-lssl". To combine all three, separate with spaces. See http://trac.greenstone.org/changeset/30948 and https://github.com/tatsuhiro-t/spdylay/issues/43
 
+PROBLEM AND SOLUTION WITH WGETRC
 Can turn off requiring a certificate check for https URLs in wgetrc conf file, as explained here:
 https://superuser.com/questions/508696/wget-without-no-check-certificate
@@ -139,5 +140,28 @@
 # https://lists.gnu.org/archive/html/bug-wget/2014-12/msg00104.html
 
-2ND PROBLEM: OpenSSL License, see https://www.openssl.org/source/license.html
+Alternatives for Windows:
+Source:
+- https://soliloquyforthefallen.net/?p=238
+- https://github.com/wertarbyte/wget/tree/master/windows (README at end)
+Binaries:
+- https://stackoverflow.com/questions/14344921/wget-for-windows-7-trusted-source
 
-QUESTION: If I delete the gs2build/bin/linux/openssl folder, the built wget does not seem to care. Is it finding something else or has it included the openssl somehow? How can I verify this?
+COMBINING GREENSTONE's GPL with OpenSSL LICENSES
+OpenSSL is under a double license, see https://www.openssl.org/source/license.html
+The licenses for GPL and OpenSSL are incompatible, see https://www.gnu.org/licenses/license-list.en.html#OpenSSL
+but you can combine it this way: https://opensource.stackexchange.com/questions/2233/gpl-v3-with-openssl-exception?rq=1
+which is what we've done for GS2 and GS3.
+
+
+TO DO:
+- If I delete the gs2build/bin/linux/openssl folder, the built wget still works fine without it. Dr Bainbridge confirmed that this is because, wget is built against OpenSSL's static libraries and therefore no longer needs the OpenSSL stuff we build and have been putting into gs2build/bin/linux/openssl. So we no longer need to put the built OpenSSL there.
+
+- Add a tick box in GLI > File > Preferences for turning on No Check Certificate over https, this should then replace our wgetrc file and env variable set in GS2's setup.bash. By default leave this flag unticked, so downloading won't work over https. Need to store this user setting in GLI's config.xml. Ensure that when the download over https failed, it results in an error.
+
+- If the downloading error count > 0:
+At the bottom of GLI > Download Pane > View Log > download error log - when we get errors:
+You have the option of adjusting your proxy server settings (go through the Configure Proxy button)
+For https certificate authentication, you have the option of turning off checking the certificate in the Connections tab of File > Preferences
+
+Check the warnings on windows. If it's no longer always warning, then do the stuff above on warning too, not just on error.
+
Index: main/trunk/greenstone2/build-src/packages/wget/wgetrc-gs.in
===================================================================
--- main/trunk/greenstone2/build-src/packages/wget/wgetrc-gs.in	(revision 31829)
+++ main/trunk/greenstone2/build-src/packages/wget/wgetrc-gs.in	(revision 31829)
@@ -0,0 +1,138 @@
+###
+### Sample Wget initialization file .wgetrc
+###
+
+# for versions with openssl support, don't want to be forced to run wget
+# with --no-check-certificate each time
+check_certificate = off
+
+## You can use this file to change the default behaviour of wget or to
+## avoid having to type many many command-line options. This file does
+## not contain a comprehensive list of commands -- look at the manual
+## to find out what you can put into this file. You can find this here:
+##   $ info wget.info 'Startup File'
+## Or online here:
+##   https://www.gnu.org/software/wget/manual/wget.html#Startup-File
+##
+## Wget initialization file can reside in /usr/local/etc/wgetrc
+## (global, for all users) or $HOME/.wgetrc (for a single user).
+##
+## To use the settings in this file, you will have to uncomment them,
+## as well as change them, in most cases, as the values on the
+## commented-out lines are the default values (e.g. "off").
+
+
+##
+## Global settings (useful for setting up in /usr/local/etc/wgetrc).
+## Think well before you change them, since they may reduce wget's
+## functionality, and make it behave contrary to the documentation:
+##
+
+# You can set retrieve quota for beginners by specifying a value
+# optionally followed by 'K' (kilobytes) or 'M' (megabytes).  The
+# default quota is unlimited.
+#quota = inf
+
+# You can lower (or raise) the default number of retries when
+# downloading a file (default is 20).
+#tries = 20
+
+# Lowering the maximum depth of the recursive retrieval is handy to
+# prevent newbies from going too "deep" when they unwittingly start
+# the recursive retrieval.  The default is 5.
+#reclevel = 5
+
+# By default Wget uses "passive FTP" transfer where the client
+# initiates the data connection to the server rather than the other
+# way around.  That is required on systems behind NAT where the client
+# computer cannot be easily reached from the Internet.  However, some
+# firewalls software explicitly supports active FTP and in fact has
+# problems supporting passive transfer.  If you are in such
+# environment, use "passive_ftp = off" to revert to active FTP.
+#passive_ftp = off
+
+# The "wait" command below makes Wget wait between every connection.
+# If, instead, you want Wget to wait only between retries of failed
+# downloads, set waitretry to maximum number of seconds to wait (Wget
+# will use "linear backoff", waiting 1 second after the first failure
+# on a file, 2 seconds after the second failure, etc. up to this max).
+#waitretry = 10
+
+
+##
+## Local settings (for a user to set in his $HOME/.wgetrc).  It is
+## *highly* undesirable to put these settings in the global file, since
+## they are potentially dangerous to "normal" users.
+##
+## Even when setting up your own ~/.wgetrc, you should know what you
+## are doing before doing so.
+##
+
+# Set this to on to use timestamping by default:
+#timestamping = off
+
+# It is a good idea to make Wget send your email address in a `From:'
+# header with your request (so that server administrators can contact
+# you in case of errors).  Wget does *not* send `From:' by default.
+#header = From: Your Name <username@site.domain>
+
+# You can set up other headers, like Accept-Language.  Accept-Language
+# is *not* sent by default.
+#header = Accept-Language: en
+
+# You can set the default proxies for Wget to use for http, https, and ftp.
+# They will override the value in the environment.
+#https_proxy = http://proxy.yoyodyne.com:18023/
+#http_proxy = http://proxy.yoyodyne.com:18023/
+#ftp_proxy = http://proxy.yoyodyne.com:18023/
+
+# If you do not want to use proxy at all, set this to off.
+#use_proxy = on
+
+# You can customize the retrieval outlook.  Valid options are default,
+# binary, mega and micro.
+#dot_style = default
+
+# Setting this to off makes Wget not download /robots.txt.  Be sure to
+# know *exactly* what /robots.txt is and how it is used before changing
+# the default!
+#robots = on
+
+# It can be useful to make Wget wait between connections.  Set this to
+# the number of seconds you want Wget to wait.
+#wait = 0
+
+# You can force creating directory structure, even if a single is being
+# retrieved, by setting this to on.
+#dirstruct = off
+
+# You can turn on recursive retrieving by default (don't do this if
+# you are not sure you know what it means) by setting this to on.
+#recursive = off
+
+# To always back up file X as X.orig before converting its links (due
+# to -k / --convert-links / convert_links = on having been specified),
+# set this variable to on:
+#backup_converted = off
+
+# To have Wget follow FTP links from HTML files by default, set this
+# to on:
+#follow_ftp = off
+
+# To try ipv6 addresses first:
+#prefer-family = IPv6
+
+# Set default IRI support state
+#iri = off
+
+# Force the default system encoding
+#locale = UTF-8
+
+# Force the default remote server encoding
+#remoteencoding = UTF-8
+
+# Turn on to prevent following non-HTTPS links when in recursive mode
+#httpsonly = off
+
+# Tune HTTPS security (auto, SSLv2, SSLv3, TLSv1, PFS)
+#secureprotocol = auto
Index: main/trunk/greenstone2/setup.bash
===================================================================
--- main/trunk/greenstone2/setup.bash	(revision 31828)
+++ main/trunk/greenstone2/setup.bash	(revision 31829)
@@ -337,2 +337,8 @@
 # Perl >= v5.18.* randomises map iteration order within a process
 export PERL_PERTURB_KEYS=0
+
+# turn off certificate errors when using wget to retrieve over https
+# (to avoid turning it off with the --no-check-certificate flag to each wget cmd)
+# See https://superuser.com/questions/508696/wget-without-no-check-certificate
+# https://www.gnu.org/software/wget/manual/html_node/Wgetrc-Location.html
+export WGETRC=$GSDLHOME/bin/$GSDLOS/wgetrc
