Changeset 34460 for main

Timestamp:

2020-10-17T17:17:49+13:00 (4 years ago)

Author:

davidb

Message:

Tidy up of code and notes

Location:

main/trunk/model-sites-dev/atea/collect/digital-nz/prepare

Files:

• 01-PREP.sh (modified) (1 diff)
• 02-RUN.sh (modified) (2 diffs)
• README.txt (modified) (1 diff)
• dnz-search-language-mi--python2.py (modified) (1 diff)
• dnz-search-language-mi--python3.py (modified) (3 diffs)

main/trunk/model-sites-dev/atea/collect/digital-nz/prepare/01-PREP.sh

@@ -30 +30 @@
 
 
-if [ ! -d import ] ; then
-    echo "Making directory: 'import'"
-    mkdir import
+if [ ! -d downloads ] ; then
+    echo "Making directory: 'downloads'"
+    mkdir downloads
 else
-    echo "Already created directory: 'import'"
+    echo "Already created directory: 'downloads'"
 fi
 

main/trunk/model-sites-dev/atea/collect/digital-nz/prepare/02-RUN.sh

@@ -17 +17 @@
 echo "****"
 echo "* Downloaded JSON records saved to: "
-echo "*    import"
+echo "*    downloads"
 echo "*"
 echo "* To remove a previous download set from collection's import folder:"
@@ -23 +23 @@
 echo "*"
 echo "* To move in a freshly downloaded set into the collection's import folder"
-echo "*   echo import/http* | xargs mv -t ../import/"
+echo "*   echo downloads/http* | xargs mv -t ../import/"
 echo "****"
 echo

main/trunk/model-sites-dev/atea/collect/digital-nz/prepare/README.txt

@@ -1 +1 @@
 
-# The following was done:
-#   12 Oct 2020
+# To setup Digital NZ's Python library code for accessing their API,
+# along with supporting files, run:
 
-git clone https://github.com/fogonwater/pydnz.git
+  ./01-PREP.sh
 
-# If, in the future, working with the latest git clone leads
-# to problems, then a fall-back position is to work with the
-# locally provided tar-ball of the code, that was taken 
-# 12 Oct 2020
-
-# Alternative version:
-#
-  tar xvzf pydnz.tar.gz
-
-# Working towards script
+# Note:
+#   Edit the start of this file to control whether or not the script
+#   clones the live github repository, or untars the snapshot of the
+#   code taken, representing a 'checkpoint' version that is known
+#   to work with the Greenstone/Atea code written that uses the
+#   Python library
 
 
 
-if [ ! -d import ] ; then
-    echo "Making directory: 'import'"
-    mkdir import
-fi
+# To access the Digtial NZ API and download JSON records which match
+#  'language=mi' run:
 
+  ./02-RUN.sh
+
+# Notes:
+#   This runs the Python3 version of the script.
+#
+#   The API returns duplicated dc_identifiers, so the number of records
+#   ingested into Greenstone is ultimately smaller than the number of
+#   records reported as matching in the Digital NZ API
+#
+#   Script utilizes the /v3/ version of the API (latest version at
+#   the time the script was being developed)
+
+
+# General Notes about the developed bespoke Python scripts that
+# call the API
+
+
+# The developed script was initially written for python2
+#
+# In a later period of development it was ported to python3
+#   Main different (other than print/print()) was how to
+#   handle UTF-8 data input/output as UTF8
+#
+# The python3 script follows the 'futures' pattern, looking
+# to be backwards compatible with python2, but this hasn't
+# been tested.
+#
+# The python2 version of the script hasn't been updated to
+# match improvements made to the python3 scripts, such as
+# reading in a key from a file, and outputing to 'downloads'
+# rather than the more confusingly named 'import', but
+# it is otherwise basically sound.
+
+# Overall ...
+#
 # The needs of the DNZ python scripts seems pretty light
-#   The system python3 most recently trialed had all the
-#   necessary packages.
+#   The system installed python3 most recently trialed had all the
+#   necessary packages.  Other than setting PYTHONPATH so it could
+#   see the code in the pydnz folder, things ran smoothly
 #
 #   If need be, virtualenv setup such as
+#
 #      virtualenv --python=python3 venv-python3-dnz
+#
 #   Combined with (from within 'pydnz):
+#
 #      pip install -r requirements.txt
+#
 #   should help
 #
-#   Note: running 'python setup.py install' in 'pydnz' resulted
-#         in SSL errors trying to download from the python/pip
-#         repository that seem to be related to a change a while
-#         back in python that broke a lot of package installers, and
-#         proved to be excessively fussy to resolve, hence the
-#         decision to go down the PYTHONPATH path route
+#   Note:
+#     Running 'python setup.py install' in 'pydnz' resulted
+#     in SSL errors trying to download from the python/pip
+#     repository that seem to be related to a change a while
+#     back in python that broke a lot of package installers, and
+#     proved to be excessively fussy to resolve (something the
+#     developers of pydnz to do???), hence the decision to go
+#     down the PYTHONPATH path route for now
+#
 
 
-export PYTHONPATH=`pwd`/pydnz
-
-./dnz-search-language-mi-python3.py
-
-
-# tidy up script
-# rewrite script to key is readin from .in file
-
-# Note: the API returns duplicated dc_identifiers
-
-# Note: At time of writing (12 Oct 2020)
-#       working with the /v3/ version of the API
-
-

main/trunk/model-sites-dev/atea/collect/digital-nz/prepare/dnz-search-language-mi--python2.py

@@ -33 +33 @@
         landing_url = unicode(landing_url).encode('utf8')
         json_landing_filename = re.sub('[:/.]', '-', landing_url) + ".json"
-        full_json_landing_filename = "import/" + json_landing_filename
+        full_json_landing_filename = "downloads/" + json_landing_filename
 
         dc_identifiers = rec["dc_identifier"]

main/trunk/model-sites-dev/atea/collect/digital-nz/prepare/dnz-search-language-mi--python3.py

@@ -7 +7 @@
 from builtins import str
 
-import dnz.api
-import pprint
-import json
+import os
 import sys
 import re
+import json
+#import pprint
+
+import dnz.api
+
+
+output_dir = "downloads"
 
 # DNZ key
@@ -49 +54 @@
         json_landing_filename  = re.sub('[.]', '~DOT~', json_landing_filename2) + ".json"
                 
-        full_json_landing_filename = "import/" + json_landing_filename
+        full_json_landing_filename = os.path.join(output_dir,json_landing_filename)
         print("json_landing_filename:\t" + json_landing_filename)
         
@@ -76 +81 @@
         print("")
 
-#    with open(json_filename, 'w') as outfile:
-#        json.dump(data, outfile)
-
 #pprint.pprint(result.records)
 
 
-#with open('data.json', 'w') as outfile:
-#    json.dump(data, outfile)