source: main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/08b-COPY-CSV-AND-AUDIO-FEATURES-TO-IMPORT_support.py@ 35990

#!/usr/bin/env python3

import argparse

import os
import re
import shutil

from glob import glob

prep_dir = "essentia-audio-features"

import_dir = os.path.join("..","import")
audio_dir  = os.path.join(prep_dir,"audio")


parser = argparse.ArgumentParser(description="Copy Essentia generated audio features into Greenstone3 collection's '../import' directory")
parser.add_argument('--startyear', type=int, default=None,
                    help='Start year range of the Eurovision Song Contest')
parser.add_argument('--endyear', type=int, default=None,
                    help='End year range of the Eurovision Song Contest')
args = parser.parse_args()

start_year = int(args.startyear) if args.startyear is not None  else None
end_year   = int(args.endyear)   if args.endyear   is not None  else None


years = [d for d in os.listdir(audio_dir) if os.path.isdir(os.path.join(audio_dir, d))]

sorted_years = sorted(years)

opt_filtered_sorted_years = []

for y in sorted_years:
     
     if (start_year is not None) and int(y) < start_year:
          continue
     
     if (end_year is not None) and int(y) > end_year:
          continue

     opt_filtered_sorted_years.append(y)
     

print("Reading in Audio-Feature JSON files by year")

all_af_json_files = []


for year in opt_filtered_sorted_years:
#     print("  " + year)
     
     if year == "2020":
          print("====")
          print("Skipping year 2020 as no contest was held that year")
          print("====")
          continue
     
     audio_year_dir = os.path.join(audio_dir,year)

     for f in os.listdir(audio_year_dir):
          if re.search(r"\.json$",f):
               year_entrant = os.path.join(year,f)
               all_af_json_files.append(year_entrant)
          

#print("json files = " + repr(all_af_json_files))


entrant_recs = []


for year_and_entrant in all_af_json_files:

     match = re.search(r"^(\d{4})/([^_]+)_([^_]+)_([^_]+)$",year_and_entrant)
          
     if match:
          year    = match.group(1)
          country = match.group(2)
          title   = match.group(3)
          artist  = match.group(4)

          entrant_rec = { "year": year, "country": country, "title": title, "artist": artist }
          

          # print("year={0}, country={1}, title={2}, artist={3}".format(year,country,title,artist))
          # print("entrant = " + repr(entrant_rec))
          
          entrant_recs.append(entrant_rec)
          
     else:
          print("**** No match found for: '" + year_and_entrant +"'")


#
# af = audio-features
#
all_af_nul_files = []
src_af_lookup    = {}

#print()
#print("Nul files derived from year/country_title_artist audio feature JSON files:")

for entrant_rec in entrant_recs:
     year    = entrant_rec["year"]
     country = entrant_rec["country"]
     title   = entrant_rec["title"]
     artist  = entrant_rec["artist"]

     full_year_entrant = os.path.join(audio_dir,year,country+"_" + title + "_" + artist)
     
     # Tweak the values so they align with those used in SPARQL resultsset     
     country = country.replace("&","and").replace(" ","")

     if year == "1956":
          title_label = entrant_rec["title"].lower().replace(" ","")
          #title_first3 = title_label[0].upper() + title_label[1:3].lower()
          title_first3 = title_label[0:3].upper()
          year += title_first3
          
     nul_file = country + year + ".nul"

     all_af_nul_files.append(nul_file)
     src_af_lookup[nul_file] = full_year_entrant
     # print("**## storing nul_file = " + nul_file + " => " + full_year_entrant);
     
# print(", ".join(all_af_nul_files))


entrant_nul_dirs = []


for d in os.listdir(import_dir):
     full_d = os.path.join(import_dir, d)
     if os.path.isdir(full_d) and re.search(r"^(sparqlresults-local--countries-in-esc-by-year-.*)|(missing-cat-countries)|(inaugural-year)$",d):
          entrant_nul_dirs.append(full_d)


#print("Entrant Nul dirs:")
#print(repr(entrant_nul_dirs))

entrant_nul_files_lookup = {}

for d in entrant_nul_dirs:

     for f in os.listdir(d):
          if re.search(r"\.nul$",f):
               nul_file = f
               full_nul_file = os.path.join(d,f)

               if not nul_file in entrant_nul_files_lookup:
                    entrant_nul_files_lookup[nul_file] = []
                    
               entrant_nul_files_lookup[nul_file].append(full_nul_file)

#print()
#print("Lookup dict:")
#print(repr(entrant_nul_files_lookup))
      


# The country of Macedonia renamed itself to North Macedonia in response
# to a long running naming debate with the region of Macedonia in Greece
# 
#
# Audio-Feature files sometimes refer to North Macedonia in places
# where the SPARQL results gives the country as Macedonia
#
# => Pass through the audio-feature data changing 'NorthMacedoniaYYYY.nul' to
#    'MacedoniaYYY.nul' for any place where there is no entrant_nul_files_lookup
#    for 'NorthMacedonia'

print()
print("Running errata to better align Audio Feature derived .nul files with Sparql Resultset")
for i,af_nul_file in enumerate(all_af_nul_files):

     if not af_nul_file in entrant_nul_files_lookup:
          af_nul_file_errata = None
          
          if af_nul_file == "Germany1956Imw.nul":
               af_nul_file_errata = "Germany1956ImW.nul"

          if af_nul_file == "Yugoslavia1992.nul":
               af_nul_file_errata = "FRYugoslavia1992.nul"
           
          elif re.search(r"^NorthMacedonia",af_nul_file):
               af_nul_file_check = af_nul_file.replace("North","")
               if af_nul_file_check in entrant_nul_files_lookup:
                    af_nul_file_errata = af_nul_file_check

          if af_nul_file_errata is not None:
               print("  Errata: Fixing " + af_nul_file + " => " + af_nul_file_errata)

               src_af_lookup[af_nul_file_errata] = src_af_lookup[af_nul_file]
               del src_af_lookup[af_nul_file]

               all_af_nul_files[i] = af_nul_file_errata
               
               
print("Matching Audio-Feature files to nul files in 'import':")


print()
print("The following files have Audio Features, but did not match in to a .nul file in the 'import' diretory")

matched_af_nul_files = []

for af_nul_file in all_af_nul_files:
     if af_nul_file in entrant_nul_files_lookup:
          matched_af_nul_files.append(af_nul_file)
     else:
          print("  " + af_nul_file)
               

print()
print("Copying the following Essentia audio feature JSON files into .nul matched positions in 'import'")

for af_nul_file in matched_af_nul_files:
     import_files = entrant_nul_files_lookup[af_nul_file]
     for import_file in import_files:
          # print("** looking up " + af_nul_file)
          src_json_file = src_af_lookup[af_nul_file]
          dst_json_file = import_file.replace(".nul","-Essentia.json")
          dst_assoc_json_file = import_file.replace(".nul","-AssocEssentia.json")

          print("  Copying: " + src_json_file + " => " + dst_json_file)
          shutil.copy(src_json_file,dst_json_file)

          print("  Copying: " + src_json_file + " => " + dst_assoc_json_file)
          shutil.copy(src_json_file,dst_assoc_json_file)
                    

# Sort out contestant CSV files to copy into ../import area

src_csv_files = []

if start_year is None and end_year is None:
     # 1956 and 1956 - env.esc_cutoff_year
     esc_cutoff_endyear = os.environ["esc_cutoff_endyear"]
     #esc_cutoff_endyear = 2021
     
     src_csv_files.append("contestants-1956.cvs")
     src_csv_files.append(f"contestants-1957-to-{esc_cutoff_endyear}.cvs")
     
elif start_year is None :
     src_csv_files.append(f"contestants-1956-to-{end_year}.cvs")

elif end_year is None :
     src_csv_files.append(f"contestants-{start_year}-to-{esc_cutoff_endyear}.cvs")
else:
     src_csv_files.append(f"contestants-{start_year}-to-{end_year}.cvs")


dst_csv_dirs = [os.path.join(import_dir,"inaugural-year"), os.path.join(import_dir,"missing-cat-countries") ]

sparql_result_dirs = glob(os.path.join(import_dir,"sparqlresults-local--countries-in-esc-by-year-*/"))
dst_csv_dirs += sparql_result_dirs


print()
print("Copying the following Contestants CSV files into 'import'")

for dst_dir in dst_csv_dirs:
     for src_csv_file in src_csv_files:

          src_csv_filename = os.path.join(prep_dir,src_csv_file)
          dst_csv_filename = os.path.join(dst_dir,src_csv_file)
          
          print("  Copying: " + src_csv_filename + " => " + dst_csv_filename)
          shutil.copy(src_csv_filename,dst_filename)