Index: /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/01-PREP.sh
===================================================================
--- /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/01-PREP.sh	(revision 34461)
+++ /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/01-PREP.sh	(revision 34461)
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+pydnz_source_opt="git"
+#pydnz_source_opt="untar"
+
+echo
+echo "****"
+echo "* pydnz_source_opt set to:"
+echo "*   '$pydnz_source_opt'"
+echo "****"
+echo
+
+if [ ! -d pydnz ] ; then
+
+    if [ $pydnz_source_opt = "git" ] ; then
+	git clone https://github.com/fogonwater/pydnz.git
+    else
+	# Assume "untar"
+	echo "Untarring pydnz.tar.gz"
+	tar xvzf pydnz.tar.gz
+    fi
+else
+    if [ $pydnz_source_opt = "git" ] ; then
+	echo "Already git cloned Python-based Digital NZ API repository: 'pydnz'"
+    else
+	# Assume "untar"
+	echo "Already untarred: pydnz.tar.gz"
+    fi
+fi
+
+
+if [ ! -d downloads ] ; then
+    echo "Making directory: 'downloads'"
+    mkdir downloads
+else
+    echo "Already created directory: 'downloads'"
+fi
+
+
+if [ ! -f dnzkey.txt ] ; then
+    echo "Generating 'dnzkey.txt' from 'dnzkey.txt.in'"
+    /bin/cp dnzkey.txt.in dnzkey.txt
+    echo 
+    echo "New edit:"
+    echo "  dnzkey.txt"
+    echo "And change <CHANGEME> to be your actual Digital NZ API key, accessed through:"   
+    echo "  https://digitalnz.org/login"
+    echo
+else
+    echo "Already generated: 'dnzkey.txt'"
+    echo "=> Assuming this has been setup with a valid Digital NZ API key"
+fi
Index: /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/02-RUN.sh
===================================================================
--- /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/02-RUN.sh	(revision 34461)
+++ /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/02-RUN.sh	(revision 34461)
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+if [ "x$PYTHONPATH" = "x" ] ; then
+    export PYTHONPATH="`pwd`/pydnz"
+else
+    export PYTHONPATH="`pwd`/pydnz:$PYTHNOPATH"
+fi
+
+echo 
+echo "Set environment variable PYTHONPATH to:"
+echo "  $PYTHONPATH"
+echo 
+
+./dnz-search-language-mi--python3.py
+
+echo
+echo "****"
+echo "* Downloaded JSON records saved to: "
+echo "*    downloads"
+echo "*"
+echo "* To remove a previous download set from collection's import folder:"
+echo "*   echo ../import/http* | xargs rm"
+echo "*"
+echo "* To move in a freshly downloaded set into the collection's import folder"
+echo "*   echo downloads/http* | xargs mv -t ../import/"
+echo "****"
+echo
Index: /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/README.txt
===================================================================
--- /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/README.txt	(revision 34461)
+++ /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/README.txt	(revision 34461)
@@ -0,0 +1,79 @@
+
+# To setup Digital NZ's Python library code for accessing their API,
+# along with supporting files, run:
+
+  ./01-PREP.sh
+
+# Note:
+#   Edit the start of this file to control whether or not the script
+#   clones the live github repository, or untars the snapshot of the
+#   code taken, representing a 'checkpoint' version that is known
+#   to work with the Greenstone/Atea code written that uses the
+#   Python library
+
+
+
+# To access the Digtial NZ API and download JSON records which match
+#  'language=mi' run:
+
+  ./02-RUN.sh
+
+# Notes:
+#   This runs the Python3 version of the script.
+#
+#   The API returns duplicated dc_identifiers, so the number of records
+#   ingested into Greenstone is ultimately smaller than the number of
+#   records reported as matching in the Digital NZ API
+#
+#   Script utilizes the /v3/ version of the API (latest version at
+#   the time the script was being developed)
+
+
+# General Notes about the developed bespoke Python scripts that
+# call the API
+
+
+# The developed script was initially written for python2
+#
+# In a later period of development it was ported to python3
+#   Main different (other than print/print()) was how to
+#   handle UTF-8 data input/output as UTF8
+#
+# The python3 script follows the 'futures' pattern, looking
+# to be backwards compatible with python2, but this hasn't
+# been tested.
+#
+# The python2 version of the script hasn't been updated to
+# match improvements made to the python3 scripts, such as
+# reading in a key from a file, and outputing to 'downloads'
+# rather than the more confusingly named 'import', but
+# it is otherwise basically sound.
+
+# Overall ...
+#
+# The needs of the DNZ python scripts seems pretty light
+#   The system installed python3 most recently trialed had all the
+#   necessary packages.  Other than setting PYTHONPATH so it could
+#   see the code in the pydnz folder, things ran smoothly
+#
+#   If need be, virtualenv setup such as
+#
+#      virtualenv --python=python3 venv-python3-dnz
+#
+#   Combined with (from within 'pydnz):
+#
+#      pip install -r requirements.txt
+#
+#   should help
+#
+#   Note:
+#     Running 'python setup.py install' in 'pydnz' resulted
+#     in SSL errors trying to download from the python/pip
+#     repository that seem to be related to a change a while
+#     back in python that broke a lot of package installers, and
+#     proved to be excessively fussy to resolve (something the
+#     developers of pydnz to do???), hence the decision to go
+#     down the PYTHONPATH path route for now
+#
+
+
Index: /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/dnz-search-language-mi--python2.py
===================================================================
--- /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/dnz-search-language-mi--python2.py	(revision 34461)
+++ /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/dnz-search-language-mi--python2.py	(revision 34461)
@@ -0,0 +1,68 @@
+#!/usr/bin/env python2
+
+import dnz.api
+import pprint
+import json
+import sys
+import re
+
+# DNZ key
+DNZ = dnz.api.Dnz('<CHANGEME>')
+
+
+#result = dnz.search('kiwi tui')
+#result = dnz.search('kiwi tui', per_page=50, page=10, fields=['id', 'title', 'collection', 'content_partner'])
+#result = dnz.search(_or={'category':['Videos', 'Images']}, _and={'content_partner':['Ministry for Culture and Heritage']})
+
+query_term={'language':['mi']}
+
+results = DNZ.search(_and=query_term, per_page=50)
+num_results=results.result_count
+print "Number of matching records: " + str(results.result_count)
+
+
+for i in range(0,num_results,50):
+    page = i / 50
+    results = DNZ.search(_and=query_term, page=page, per_page=50)
+
+    for rec in results.records:
+        #    pprint.pprint(rec)
+        landing_url = rec["landing_url"]
+        print "Landing URL: " + rec["landing_url"]
+
+        landing_url = unicode(landing_url).encode('utf8')
+        json_landing_filename = re.sub('[:/.]', '-', landing_url) + ".json"
+        full_json_landing_filename = "downloads/" + json_landing_filename
+
+        dc_identifiers = rec["dc_identifier"]
+
+        #json.dump(rec, sys.stdout)
+
+        if (len(dc_identifiers)>0):
+                        
+            full_id = unicode(dc_identifiers[0]).encode('utf8')
+            print "  ID: " + full_id
+
+            # tail_id = full_id.replace('http://hdl.handle.net/','')
+        
+            # json_filename = re.sub('[:/.]', '-', tail_id) + ".json"
+            # print json_filename
+        else:
+            print "**** No ID specified"
+
+
+        with open(full_json_landing_filename, 'w') as outfile:
+            json.dump(rec, outfile)
+            
+        print ""
+
+#    with open(json_filename, 'w') as outfile:
+#        json.dump(data, outfile)
+
+#pprint.pprint(result.records)
+
+
+#with open('data.json', 'w') as outfile:
+#    json.dump(data, outfile)
+
+
Index: /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/dnz-search-language-mi--python3.py
===================================================================
--- /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/dnz-search-language-mi--python3.py	(revision 34461)
+++ /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/dnz-search-language-mi--python3.py	(revision 34461)
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from builtins import str
+
+import os
+import sys
+import re
+import json
+#import pprint
+
+import dnz.api
+
+
+output_dir = "downloads"
+
+# DNZ key
+with open('dnzkey.txt','r') as file:
+    dnzkey_str = file.read().strip()
+    
+# print("dnzkey_str = '" + dnzkey_str + "'")
+DNZ = dnz.api.Dnz(dnzkey_str) 
+
+
+#result = dnz.search('kiwi tui')
+#result = dnz.search('kiwi tui', per_page=50, page=10, fields=['id', 'title', 'collection', 'content_partner'])
+#result = dnz.search(_or={'category':['Videos', 'Images']}, _and={'content_partner':['Ministry for Culture and Heritage']})
+
+query_term={'language':['mi']}
+
+results = DNZ.search(_and=query_term, per_page=50)
+num_results=results.result_count
+print("Number of matching records: " + str(results.result_count))
+
+
+for i in range(0,num_results,50):
+    page = i / 50
+    results = DNZ.search(_and=query_term, page=page, per_page=50)
+
+    for rec in results.records:
+        #    pprint.pprint(rec)
+        landing_url = rec["landing_url"]
+        print("Landing URL: " + rec["landing_url"])
+
+        ### !!! landing_url = unicode(landing_url).encode('utf8')
+        ##landing_url = str(landing_url).encode('utf8')
+        #json_landing_filename = re.sub('[:/.]', '-', landing_url) + ".json"
+
+        json_landing_filename1 = re.sub('[:]', '~COLON~', landing_url)
+        json_landing_filename2 = re.sub('[/]', '~SLASH~', json_landing_filename1)
+        json_landing_filename  = re.sub('[.]', '~DOT~', json_landing_filename2) + ".json"
+                
+        full_json_landing_filename = os.path.join(output_dir,json_landing_filename)
+        print("json_landing_filename:\t" + json_landing_filename)
+        
+        dc_identifiers = rec["dc_identifier"]
+
+        #json.dump(rec, sys.stdout)
+
+        if (len(dc_identifiers)>0):
+                        
+            ### !!! full_id = unicode(dc_identifiers[0]).encode('utf8')
+            full_id = dc_identifiers[0]
+            
+            print("  ID: " + full_id)
+
+            # tail_id = full_id.replace('http://hdl.handle.net/','')
+        
+            # json_filename = re.sub('[:/.]', '-', tail_id) + ".json"
+            # print json_filename
+        else:
+            print("**** No ID specified")
+
+
+        with open(full_json_landing_filename, 'w') as outfile:
+            json.dump(rec, outfile)
+            
+        print("")
+
+#pprint.pprint(result.records)
+
+
+
+
Index: /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/dnzkey.txt.in
===================================================================
--- /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/dnzkey.txt.in	(revision 34461)
+++ /main/trunk/model-sites-dev/atea/collect/digital-nz/pre-import/dnzkey.txt.in	(revision 34461)
@@ -0,0 +1,1 @@
+<CHANGEME>
