Index: main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/08b-COPY-AUDIO-FEATURES-TO-IMPORT.py
===================================================================
--- main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/08b-COPY-AUDIO-FEATURES-TO-IMPORT.py	(revision 35984)
+++ 	(revision )
@@ -1,229 +1,0 @@
-#!/usr/bin/env python3
-
-import argparse
-
-import os
-import re
-import shutil
-
-prep_dir = "essentia-audio-features"
-
-import_dir = os.path.join("..","import")
-audio_dir  = os.path.join(prep_dir,"audio")
-
-
-parser = argparse.ArgumentParser(description="Copy Essentia generated audio features into Greenstone3 collection's '../import' directory")
-parser.add_argument('--startyear', type=int, default=None,
-                    help='Start year range of the Eurovision Song Contest')
-parser.add_argument('--endyear', type=int, default=None,
-                    help='End year range of the Eurovision Song Contest')
-args = parser.parse_args()
-
-start_year = int(args.startyear) if args.startyear is not None  else None
-end_year   = int(args.endyear)   if args.endyear   is not None  else None
-
-
-years = [d for d in os.listdir(audio_dir) if os.path.isdir(os.path.join(audio_dir, d))]
-
-sorted_years = sorted(years)
-
-opt_filtered_sorted_years = []
-
-for y in sorted_years:
-     
-     if (start_year is not None) and int(y) < start_year:
-          continue
-     
-     if (end_year is not None) and int(y) > end_year:
-          continue
-
-     opt_filtered_sorted_years.append(y)
-     
-
-print("Reading in Audio-Feature JSON files by year")
-
-all_af_json_files = []
-
-
-for year in opt_filtered_sorted_years:
-#     print("  " + year)
-     
-     if year == "2020":
-          print("====")
-          print("Skipping year 2020 as no contest was held that year")
-          print("====")
-          continue
-     
-     audio_year_dir = os.path.join(audio_dir,year)
-
-     for f in os.listdir(audio_year_dir):
-          if re.search(r"\.json$",f):
-               year_entrant = os.path.join(year,f)
-               all_af_json_files.append(year_entrant)
-          
-
-#print("json files = " + repr(all_af_json_files))
-
-
-entrant_recs = []
-
-
-for year_and_entrant in all_af_json_files:
-
-     match = re.search(r"^(\d{4})/([^_]+)_([^_]+)_([^_]+)$",year_and_entrant)
-          
-     if match:
-          year    = match.group(1)
-          country = match.group(2)
-          title   = match.group(3)
-          artist  = match.group(4)
-
-          entrant_rec = { "year": year, "country": country, "title": title, "artist": artist }
-          
-
-          # print("year={0}, country={1}, title={2}, artist={3}".format(year,country,title,artist))
-          # print("entrant = " + repr(entrant_rec))
-          
-          entrant_recs.append(entrant_rec)
-          
-     else:
-          print("**** No match found for: '" + year_and_entrant +"'")
-
-
-#
-# af = audio-features
-#
-all_af_nul_files = []
-src_af_lookup    = {}
-
-#print()
-#print("Nul files derived from year/country_title_artist audio feature JSON files:")
-
-for entrant_rec in entrant_recs:
-     year    = entrant_rec["year"]
-     country = entrant_rec["country"]
-     title   = entrant_rec["title"]
-     artist  = entrant_rec["artist"]
-
-     full_year_entrant = os.path.join(audio_dir,year,country+"_" + title + "_" + artist)
-     
-     # Tweak the values so they align with those used in SPARQL resultsset     
-     country = country.replace("&","and").replace(" ","")
-
-     if year == "1956":
-          title_label = entrant_rec["title"].lower().replace(" ","")
-          title_first3 = title_label[0].upper() + title_label[1:3].lower()
-          year += title_first3
-          
-     nul_file = country + year + ".nul"
-
-     all_af_nul_files.append(nul_file)
-     src_af_lookup[nul_file] = full_year_entrant
-     # print("**## storing nul_file = " + nul_file + " => " + full_year_entrant);
-     
-# print(", ".join(all_af_nul_files))
-
-
-entrant_nul_dirs = []
-
-
-for d in os.listdir(import_dir):
-     full_d = os.path.join(import_dir, d)
-     if os.path.isdir(full_d) and re.search(r"^(sparqlresults-local--countries-in-esc-by-year-.*)|(missing-cat-countries)|(inaugural-year)$",d):
-          entrant_nul_dirs.append(full_d)
-
-
-#print("Entrant Nul dirs:")
-#print(repr(entrant_nul_dirs))
-
-entrant_nul_files_lookup = {}
-
-for d in entrant_nul_dirs:
-
-     for f in os.listdir(d):
-          if re.search(r"\.nul$",f):
-               nul_file = f
-               full_nul_file = os.path.join(d,f)
-
-               if not nul_file in entrant_nul_files_lookup:
-                    entrant_nul_files_lookup[nul_file] = []
-                    
-               entrant_nul_files_lookup[nul_file].append(full_nul_file)
-
-#print()
-#print("Lookup dict:")
-#print(repr(entrant_nul_files_lookup))
-      
-
-
-# The country of Macedonia renamed itself to North Macedonia in response
-# to a long running naming debate with the region of Macedonia in Greece
-# 
-#
-# Audio-Feature files sometimes refer to North Macedonia in places
-# where the SPARQL results gives the country as Macedonia
-#
-# => Pass through the audio-feature data changing 'NorthMacedoniaYYYY.nul' to
-#    'MacedoniaYYY.nul' for any place where there is no entrant_nul_files_lookup
-#    for 'NorthMacedonia'
-
-print()
-print("Running errata to better align Audio Feature derived .nul files with Sparql Resultset")
-for i,af_nul_file in enumerate(all_af_nul_files):
-
-     if not af_nul_file in entrant_nul_files_lookup:
-          af_nul_file_errata = None
-          
-          if af_nul_file == "Germany1956Imw.nul":
-               af_nul_file_errata = "Germany1956ImW.nul"
-
-          if af_nul_file == "Yugoslavia1992.nul":
-               af_nul_file_errata = "FRYugoslavia1992.nul"
-           
-          elif re.search(r"^NorthMacedonia",af_nul_file):
-               af_nul_file_check = af_nul_file.replace("North","")
-               if af_nul_file_check in entrant_nul_files_lookup:
-                    af_nul_file_errata = af_nul_file_check
-
-          if af_nul_file_errata is not None:
-               print("  Errata: Fixing " + af_nul_file + " => " + af_nul_file_errata)
-
-               src_af_lookup[af_nul_file_errata] = src_af_lookup[af_nul_file]
-               del src_af_lookup[af_nul_file]
-
-               all_af_nul_files[i] = af_nul_file_errata
-               
-               
-print("Matching Audio-Feature files to nul files in 'import':")
-
-
-print()
-print("The following files have Audio Features, but did not match in to a .nul file in the 'import' diretory")
-
-matched_af_nul_files = []
-
-for af_nul_file in all_af_nul_files:
-     if af_nul_file in entrant_nul_files_lookup:
-          matched_af_nul_files.append(af_nul_file)
-     else:
-          print("  " + af_nul_file)
-               
-
-print()
-print("Copying the following Essentia audio feature JSON files into .nul matched positions in 'import'")
-
-for af_nul_file in matched_af_nul_files:
-     import_files = entrant_nul_files_lookup[af_nul_file]
-     for import_file in import_files:
-          # print("** looking up " + af_nul_file)
-          src_json_file = src_af_lookup[af_nul_file]
-          dst_json_file = import_file.replace(".nul","-Essentia.json")
-          dst_assoc_json_file = import_file.replace(".nul","-AssocEssentia.json")
-
-          print("  Copying: " + src_json_file + " => " + dst_json_file)
-          shutil.copy(src_json_file,dst_json_file)
-
-          print("  Copying: " + src_json_file + " => " + dst_assoc_json_file)
-          shutil.copy(src_json_file,dst_assoc_json_file)
-                    
-
Index: main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/08b-COPY-CSV-AND-AUDIO-FEATURES-TO-IMPORT_support.py
===================================================================
--- main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/08b-COPY-CSV-AND-AUDIO-FEATURES-TO-IMPORT_support.py	(revision 35985)
+++ main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/08b-COPY-CSV-AND-AUDIO-FEATURES-TO-IMPORT_support.py	(revision 35985)
@@ -0,0 +1,266 @@
+#!/usr/bin/env python3
+
+import argparse
+
+import os
+import re
+import shutil
+
+from glob import glob
+
+prep_dir = "essentia-audio-features"
+
+import_dir = os.path.join("..","import")
+audio_dir  = os.path.join(prep_dir,"audio")
+
+
+parser = argparse.ArgumentParser(description="Copy Essentia generated audio features into Greenstone3 collection's '../import' directory")
+parser.add_argument('--startyear', type=int, default=None,
+                    help='Start year range of the Eurovision Song Contest')
+parser.add_argument('--endyear', type=int, default=None,
+                    help='End year range of the Eurovision Song Contest')
+args = parser.parse_args()
+
+start_year = int(args.startyear) if args.startyear is not None  else None
+end_year   = int(args.endyear)   if args.endyear   is not None  else None
+
+
+years = [d for d in os.listdir(audio_dir) if os.path.isdir(os.path.join(audio_dir, d))]
+
+sorted_years = sorted(years)
+
+opt_filtered_sorted_years = []
+
+for y in sorted_years:
+     
+     if (start_year is not None) and int(y) < start_year:
+          continue
+     
+     if (end_year is not None) and int(y) > end_year:
+          continue
+
+     opt_filtered_sorted_years.append(y)
+     
+
+print("Reading in Audio-Feature JSON files by year")
+
+all_af_json_files = []
+
+
+for year in opt_filtered_sorted_years:
+#     print("  " + year)
+     
+     if year == "2020":
+          print("====")
+          print("Skipping year 2020 as no contest was held that year")
+          print("====")
+          continue
+     
+     audio_year_dir = os.path.join(audio_dir,year)
+
+     for f in os.listdir(audio_year_dir):
+          if re.search(r"\.json$",f):
+               year_entrant = os.path.join(year,f)
+               all_af_json_files.append(year_entrant)
+          
+
+#print("json files = " + repr(all_af_json_files))
+
+
+entrant_recs = []
+
+
+for year_and_entrant in all_af_json_files:
+
+     match = re.search(r"^(\d{4})/([^_]+)_([^_]+)_([^_]+)$",year_and_entrant)
+          
+     if match:
+          year    = match.group(1)
+          country = match.group(2)
+          title   = match.group(3)
+          artist  = match.group(4)
+
+          entrant_rec = { "year": year, "country": country, "title": title, "artist": artist }
+          
+
+          # print("year={0}, country={1}, title={2}, artist={3}".format(year,country,title,artist))
+          # print("entrant = " + repr(entrant_rec))
+          
+          entrant_recs.append(entrant_rec)
+          
+     else:
+          print("**** No match found for: '" + year_and_entrant +"'")
+
+
+#
+# af = audio-features
+#
+all_af_nul_files = []
+src_af_lookup    = {}
+
+#print()
+#print("Nul files derived from year/country_title_artist audio feature JSON files:")
+
+for entrant_rec in entrant_recs:
+     year    = entrant_rec["year"]
+     country = entrant_rec["country"]
+     title   = entrant_rec["title"]
+     artist  = entrant_rec["artist"]
+
+     full_year_entrant = os.path.join(audio_dir,year,country+"_" + title + "_" + artist)
+     
+     # Tweak the values so they align with those used in SPARQL resultsset     
+     country = country.replace("&","and").replace(" ","")
+
+     if year == "1956":
+          title_label = entrant_rec["title"].lower().replace(" ","")
+          #title_first3 = title_label[0].upper() + title_label[1:3].lower()
+          title_first3 = title_label[0:3].upper()
+          year += title_first3
+          
+     nul_file = country + year + ".nul"
+
+     all_af_nul_files.append(nul_file)
+     src_af_lookup[nul_file] = full_year_entrant
+     # print("**## storing nul_file = " + nul_file + " => " + full_year_entrant);
+     
+# print(", ".join(all_af_nul_files))
+
+
+entrant_nul_dirs = []
+
+
+for d in os.listdir(import_dir):
+     full_d = os.path.join(import_dir, d)
+     if os.path.isdir(full_d) and re.search(r"^(sparqlresults-local--countries-in-esc-by-year-.*)|(missing-cat-countries)|(inaugural-year)$",d):
+          entrant_nul_dirs.append(full_d)
+
+
+#print("Entrant Nul dirs:")
+#print(repr(entrant_nul_dirs))
+
+entrant_nul_files_lookup = {}
+
+for d in entrant_nul_dirs:
+
+     for f in os.listdir(d):
+          if re.search(r"\.nul$",f):
+               nul_file = f
+               full_nul_file = os.path.join(d,f)
+
+               if not nul_file in entrant_nul_files_lookup:
+                    entrant_nul_files_lookup[nul_file] = []
+                    
+               entrant_nul_files_lookup[nul_file].append(full_nul_file)
+
+#print()
+#print("Lookup dict:")
+#print(repr(entrant_nul_files_lookup))
+      
+
+
+# The country of Macedonia renamed itself to North Macedonia in response
+# to a long running naming debate with the region of Macedonia in Greece
+# 
+#
+# Audio-Feature files sometimes refer to North Macedonia in places
+# where the SPARQL results gives the country as Macedonia
+#
+# => Pass through the audio-feature data changing 'NorthMacedoniaYYYY.nul' to
+#    'MacedoniaYYY.nul' for any place where there is no entrant_nul_files_lookup
+#    for 'NorthMacedonia'
+
+print()
+print("Running errata to better align Audio Feature derived .nul files with Sparql Resultset")
+for i,af_nul_file in enumerate(all_af_nul_files):
+
+     if not af_nul_file in entrant_nul_files_lookup:
+          af_nul_file_errata = None
+          
+          if af_nul_file == "Germany1956Imw.nul":
+               af_nul_file_errata = "Germany1956ImW.nul"
+
+          if af_nul_file == "Yugoslavia1992.nul":
+               af_nul_file_errata = "FRYugoslavia1992.nul"
+           
+          elif re.search(r"^NorthMacedonia",af_nul_file):
+               af_nul_file_check = af_nul_file.replace("North","")
+               if af_nul_file_check in entrant_nul_files_lookup:
+                    af_nul_file_errata = af_nul_file_check
+
+          if af_nul_file_errata is not None:
+               print("  Errata: Fixing " + af_nul_file + " => " + af_nul_file_errata)
+
+               src_af_lookup[af_nul_file_errata] = src_af_lookup[af_nul_file]
+               del src_af_lookup[af_nul_file]
+
+               all_af_nul_files[i] = af_nul_file_errata
+               
+               
+print("Matching Audio-Feature files to nul files in 'import':")
+
+
+print()
+print("The following files have Audio Features, but did not match in to a .nul file in the 'import' diretory")
+
+matched_af_nul_files = []
+
+for af_nul_file in all_af_nul_files:
+     if af_nul_file in entrant_nul_files_lookup:
+          matched_af_nul_files.append(af_nul_file)
+     else:
+          print("  " + af_nul_file)
+               
+
+print()
+print("Copying the following Essentia audio feature JSON files into .nul matched positions in 'import'")
+
+for af_nul_file in matched_af_nul_files:
+     import_files = entrant_nul_files_lookup[af_nul_file]
+     for import_file in import_files:
+          # print("** looking up " + af_nul_file)
+          src_json_file = src_af_lookup[af_nul_file]
+          dst_json_file = import_file.replace(".nul","-Essentia.json")
+          dst_assoc_json_file = import_file.replace(".nul","-AssocEssentia.json")
+
+          print("  Copying: " + src_json_file + " => " + dst_json_file)
+#          shutil.copy(src_json_file,dst_json_file)
+
+          print("  Copying: " + src_json_file + " => " + dst_assoc_json_file)
+#          shutil.copy(src_json_file,dst_assoc_json_file)
+                    
+
+# Sort out contestant CSV files to copy into ../import area
+
+src_csv_filenames = []
+
+if start_year is None and end_year is None:
+     # 1956 and 1956 - env.esc_cutoff_year
+     esc_cutoff_endyear = os.environ["esc_cutoff_endyear"]
+     #esc_cutoff_endyear = 2021
+     
+     src_csv_filenames.append("contestants-1956.cvs")
+     src_csv_filenames.append(f"contestants-1957-to-{esc_cutoff_endyear}.cvs")
+     
+elif start_year is None :
+     src_csv_filenames.append(f"contestants-1956-to-{end_year}.cvs")
+
+elif end_year is None :
+     src_csv_filenames.append(f"contestants-{start_year}-to-{esc_cutoff_endyear}.cvs")
+else:
+     src_csv_filenames.append(f"contestants-{start_year}-to-{end_year}.cvs")
+
+
+dst_csv_dirs = [os.path.join(import_dir,"inaugural-year"), os.path.join(import_dir,"missing-cat-countries") ]
+
+
+sparql_result_dirs = glob(os.path.join(import_dir,"sparqlresults-local--countries-in-esc-by-year-*/"))
+
+dst_csv_dirs += sparql_result_dirs
+
+print("****** src_csv_filename = " + repr(src_csv_filenames))
+
+print("****** dst_csv_dirs = " + repr(dst_csv_dirs))
+
+
+# cp contestants-2015.csv ../../import/sparqlresults-local--countries-in-esc-by-year-just-2015--with-errata/metadata-contestants-2015.csv 
