Index: main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/README.md
===================================================================
--- main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/README.md	(revision 35141)
+++ main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/README.md	(revision 35141)
@@ -0,0 +1,16 @@
+Simple experimental Eurovision-Music-Information-Grabber.
+
+# To install:
+1. `pip install -r requirements.txt`
+2. Need to set `SPOTIPY_CLIENT_ID` and `SPOTIPY_CLIENT_SECRET` environment variables! See https://developer.spotify.com/documentation/general/guides/authorization-guide/#to-obtain-authorization to obtain some.
+
+# To run: 
+`python3 ./eurosparqlify.py`
+
+# What it does:
+1. Select songs from the Eurovision LOD DL based on `<http://greenstone.org/gsextracted#song_robust_uri>` URIs.
+2. Select artist name and song title (English literals) from corresponding dbpedia entries.
+3. Search Spotify using these names and titles as query parameters. Retrieve corresponding ISRC codes, 30 sec preview clips, audio features, audio analyses. See e.g. https://developer.spotify.com/documentation/web-api/reference/#endpoint-get-audio-analysis
+4. (WIP) Retrieve further metadata from Musicbrainz / Acousticbrainz, based on ISRC codes.
+
+
Index: main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/RUN.sh
===================================================================
--- main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/RUN.sh	(revision 35141)
+++ main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/RUN.sh	(revision 35141)
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+python3 ./eurosparqlify.py \
+  --musicbrainz \
+  --data-out esc-mir-gold-data.out \
+  --mapping-out esc-mir-gold-mapping.out \
+  --rdf-out cached--esc-mir-gold-rdf.ttl
+
Index: main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/eurosparqlify.py
===================================================================
--- main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/eurosparqlify.py	(revision 35141)
+++ main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/eurosparqlify.py	(revision 35141)
@@ -0,0 +1,247 @@
+import re
+import argparse
+import json
+import csv
+
+import musicbrainzngs
+from urllib.parse import quote 
+from time import sleep
+
+import spotipy
+from spotipy.oauth2 import SpotifyClientCredentials
+from SPARQLWrapper import SPARQLWrapper, JSON, POST
+
+from rdflib import RDF, FOAF, Namespace, Graph, URIRef, Literal, BNode
+
+from pprint import pprint
+
+import os
+
+euroEndpoint = "https://so-we-must-think.space/greenstone3-lod3/greenstone/query" 
+dbpEndpoint = "https://dbpedia.org/sparql"
+
+musicbrainzngs.set_useragent(app="LinkedEurovision", version="0.01", contact="weigl@mdw.ac.at")
+
+def retrieveKey(key, mode): 
+    # given a Spotify key (see https://en.wikipedia.org/wiki/Pitch_class#Integer_notation)
+    # and a Spotify mode (0 == minor, 1 == major)
+    # return an enharmonically dubious guess using the Keys ontology
+    keys = {
+        0: "C", 
+        1: "CSharp",
+        2: "D", 
+        3: "DSharp",
+        4: "E",
+        5: "F",
+        6: "FSharp",
+        7: "G",
+        8: "GSharp",
+        9: "A", 
+        10: "ASharp",
+        11: "B"
+    }
+    modes = { 
+        0: "Minor",
+        1: "Major"
+    }
+    return "http://motools.sourceforge.net/keys/keys.owl#" + keys[key] + modes[mode]
+
+def get_dbp_results():
+    euroSparql = SPARQLWrapper(euroEndpoint)
+    euroSparql.setReturnFormat(JSON)
+    dbpSparql = SPARQLWrapper(dbpEndpoint)
+    dbpSparql.setReturnFormat(JSON)
+
+    # how many entry - song pairings are there?
+    euroSparql.setQuery("""
+      SELECT (COUNT(DISTINCT *) as ?count) WHERE {
+        GRAPH <https://so-we-must-think.space/greenstone3/eurovision-library/collection/eurovision> {
+          ?entryUri <http://greenstone.org/gsdlextracted#song_robust_uri> ?songUri .
+        }
+      } 
+    """)
+    countResult = euroSparql.query().convert()
+    count = int(countResult["results"]["bindings"][0]["count"]["value"])
+
+    # retrieve entry-song pairings and corresponding dbpedia matches IN CHUNKS OF 500
+    n=0
+    dbpBindings = []
+    while n < count:
+        euroSparql.setQuery("""
+          SELECT DISTINCT ?entryUri ?songUri WHERE {
+            graph <https://so-we-must-think.space/greenstone3/eurovision-library/collection/eurovision> {
+              ?entryUri <http://greenstone.org/gsdlextracted#song_robust_uri> ?songUri .
+            }
+          } LIMIT 500 OFFSET """ + str(n) + """
+        """)
+
+        print("Issuing Greenstone SPARQL query with n=", n)
+        songUris = euroSparql.query().convert()
+        print("... done")
+        
+        dbpValues = ""
+        for result in songUris["results"]["bindings"]:
+            dbpValues = dbpValues + " (<" + result["entryUri"]["value"] + "> <"+ result["songUri"]["value"]+ ">)"
+            
+        print("Issuing DBpedia SPARQL query ...")
+        query = """
+          SELECT DISTINCT ?entryUri ?songUri ?artist ?artistName ?title ?year ?artistMbz WHERE {
+            VALUES (?entryUri ?songUri) { 
+          """ + dbpValues + """
+            }
+            ?songUri <http://www.w3.org/2000/01/rdf-schema#label> ?title ;
+                     <http://dbpedia.org/property/artist> ?artist ;
+                     <http://dbpedia.org/property/year> ?year .
+            OPTIONAL { 
+                ?artist <http://www.w3.org/2000/01/rdf-schema#label> ?artistLabel .
+            }
+            OPTIONAL { 
+                ?artist <http://www.w3.org/2002/07/owl#sameAs> ?ext.
+                FILTER(CONTAINS(STR(?ext), "musicbrainz.org")) .
+            }
+            BIND(COALESCE(?artistLabel, ?artist) as ?artistName) .                           
+            BIND(COALESCE(?ext, "no_artist_mbz") as ?artistMbz) .
+            FILTER(LANG(?title) = "en") .
+            FILTER(LANG(?artistName) = "en") .
+          }
+          """
+        print(query)
+        dbpSparql.setQuery(query)
+        dbpSparql.setMethod(POST)
+        dbpResults = dbpSparql.query().convert()
+        dbpBindings += dbpResults["results"]["bindings"]
+        n += 500;  # move the offset window for the greenstone query
+    print("... done")
+
+    return dbpBindings
+
+def process_dbp_results(dbpResults, query_musicbrainz, output_audio_analysis):
+    g = Graph()
+    mo = Namespace("http://purl.org/ontology/mo/")
+
+    metadata_records = []
+    
+    sp = spotipy.Spotify(auth_manager=SpotifyClientCredentials())
+
+    songTuples = []
+
+    identifier_mapping = []
+
+    for result in dbpResults:
+        songTuples.append((
+            re.sub(" \(.*singer\)$", "", result["artistName"]["value"]), 
+            re.sub(" \(.*song\)$", "", result["title"]["value"]), 
+            result["songUri"]["value"], 
+            result["artist"]["value"] if result["artist"]["type"] == "uri" else "",
+            result["artistMbz"]["value"],
+            result["entryUri"]["value"]
+            ))
+
+    for tup in songTuples:
+        metadata_record = {}
+        
+        print("artist:{0} track:{1}".format(tup[0], tup[1]))
+        metadata_record["artist"] = tup[0]
+        metadata_record["track"]  = tup[1]
+
+        track_mapping_record_template = []
+        track_mapping_record_template.append(tup[0]) # spotify artist name
+        track_mapping_record_template.append(tup[1]) # spotify track title
+        track_mapping_record_template.append(tup[2]) # dbp song uri
+        track_mapping_record_template.append(tup[3]) # dbp artist uri (if available, else "")
+        track_mapping_record_template.append(tup[4]) # mbz artist uri (if available, else "")
+        track_mapping_record_template.append(tup[5]) # LinkedEurovision Entry URI
+        
+        spotifyResults = sp.search(q="artist:{0} track:{1}".format(tup[0], tup[1]))
+        tracks = spotifyResults["tracks"]["items"]
+
+        print("Number of matching artist-tracks found in Spotify: {0}".format(len(tracks)));
+
+        spotify_metadata_records = []
+        
+        for track in tracks:
+            spotify_metadata_record = {}
+            track_mapping_record = track_mapping_record_template.copy()
+            print("  Found spotify match: {0}: {1}".format(track["external_ids"], track["preview_url"]))
+            spotify_metadata_record["external_ids"] = track["external_ids"]
+            spotify_metadata_record["preview_url"]  = track["preview_url"]
+            spotify_metadata_record["audio_features"] = sp.audio_features(track["uri"])
+            print("  Audio features: ", spotify_metadata_record["audio_features"])
+            if (output_audio_analysis):
+                print("  Appending audio-analysis to spotify-metadata-record")
+                spotify_metadata_record["audio_analysis"] = sp.audio_analysis(track["uri"])
+
+            track_mapping_record.append(track["external_ids"]["isrc"] if "isrc" in track["external_ids"] else "") # isrc (if available, else "")
+            track_mapping_record.append("") # placeholder for mbz record id if available and requested by user
+            if (query_musicbrainz):
+                try:
+                    mbz = musicbrainzngs.get_recordings_by_isrc(track["external_ids"]["isrc"], includes="artists")
+                    print("  Appending MusicBrainz data to spotify-metadata-record")
+                    spotify_metadata_record["mbz"] = mbz
+                    track_mapping_record[7] = mbz 
+                    # Alignment check -- does our artist mbzid from dbpedia match the one attached to this ISRC?
+                    artist_mbz_id = track_mapping_record[4].replace("http://musicbrainz.org/artist/", "")
+                    if mbz != "no_artist_mbz" and str(mbz).rfind(artist_mbz_id) >= 0:
+                        # Golden alignment match!!
+                        # add spotify feature metadata to musical work 
+                        g.add(( URIRef(track_mapping_record[2]), mo.bpm, Literal(int(spotify_metadata_record["audio_features"][0]["tempo"])) ))
+                        g.add(( URIRef(track_mapping_record[2]), mo.key, URIRef(retrieveKey(spotify_metadata_record["audio_features"][0]["key"], spotify_metadata_record["audio_features"][0]["mode"])) ))
+                        g.add(( URIRef(track_mapping_record[2]), mo.meter, Literal(spotify_metadata_record["audio_features"][0]["time_signature"]) ))
+                        # add preview and isrc to ENTRY -- should really be on RECORD which hangs off SIGNAL and PERFORMANCE
+                        if "preview" in spotify_metadata_record:
+                            g.add(( URIRef(track_mapping_record[5]), mo.preview, URIRef(spotify_metadata_record["preview_url"]) ))
+                        if(track_mapping_record[6] != ""):
+                            g.add(( URIRef(track_mapping_record[5]), mo.isrc, URIRef("http://musicbrainz.org/isrc/" + track_mapping_record[6]) ))
+
+                        # add metadata to ARTIST 
+                        g.add(( URIRef(track_mapping_record[3]), mo.musicbrainz, URIRef(track_mapping_record[4]) ))
+                        # TODO Artist Wikipedia page, to crossref dbpedia => Wikipedia <= mbz
+
+                except musicbrainzngs.MusicBrainzError: 
+                    print("Couldn't retrieve track from Musicbrainz!")
+                # pprint(mbz)
+
+            spotify_metadata_records.append(spotify_metadata_record)            
+            identifier_mapping.append(track_mapping_record)
+            sleep(0.1)
+
+        print("")
+
+        metadata_record["spotify_tracks"] = spotify_metadata_records        
+        metadata_records.append(metadata_record)
+
+    return [metadata_records, identifier_mapping, g]
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-mb', '--musicbrainz',   action='store_true', help="Query the MusicBrainz API")
+    parser.add_argument('-aa', '--audioanalysis', action='store_true', help="Include audio analysis features in output data")
+    parser.add_argument('-out', '--data-out', help="Filename for data output (json)")
+    parser.add_argument('-mapping', '--mapping-out', help="Filename for identifier mapping output (tsv)")
+    parser.add_argument('-rdf', '--rdf-out', help="Filename for RDF (turtle) output (.ttl)")
+    
+    args = parser.parse_args()
+
+    query_musicbrainz     = getattr(args,'musicbrainz');
+    output_audio_analysis = getattr(args,'audioanalysis');
+    
+    json_output_filename = getattr(args,'data_out');
+    tsv_mapping_output_filename = getattr(args,'mapping_out');
+    rdf_output_filename = getattr(args,'rdf_out');
+
+    
+    dbp_results = get_dbp_results();
+    outcomes = process_dbp_results(dbp_results, query_musicbrainz, output_audio_analysis)
+    metadata_records = outcomes[0]
+    mapping = outcomes[1]
+    rdf = outcomes[2]
+    if json_output_filename: 
+        with open(json_output_filename, 'w') as out:
+            json.dump(metadata_records, out, indent=2)
+    if tsv_mapping_output_filename:
+        with open(tsv_mapping_output_filename, 'w') as out:
+            writer = csv.writer(out, delimiter="\t")
+            writer.writerows(mapping)
+    if rdf_output_filename:
+        rdf.serialize(rdf_output_filename, format="turtle")
Index: main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/local--esc-mir-gold.sparql
===================================================================
--- main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/local--esc-mir-gold.sparql	(revision 35141)
+++ main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/local--esc-mir-gold.sparql	(revision 35141)
@@ -0,0 +1,22 @@
+PREFIX gsdlextracted: <http://greenstone.org/gsdlextracted#>
+PREFIX mo: <http://purl.org/ontology/mo/>
+
+SELECT (?song_robust_str AS ?song_robust) (?year AS ?Year) (?country AS ?Country) (?key AS ?MOKey) (?meter as ?MOMeter) (?tempo AS ?MOTempo)
+WHERE {
+  GRAPH <https://so-we-must-think.space/greenstone3/eurovision-library/collection/eurovision> {
+
+    ?esc_entrant_uri gsdlextracted:Year ?year.
+    ?esc_entrant_uri gsdlextracted:Country ?country.
+    ?esc_entrant_uri gsdlextracted:song_robust_uri ?song_robust_str.
+  }
+  BIND(IRI(?song_robust_str) AS ?song_robust_uri).
+
+  GRAPH <https://so-we-must-think.space/greenstone3/eurovision-library/collection/eurovision-mir> {
+    ?song_robust_uri <http://purl.org/ontology/mo/key> ?key.
+    ?song_robust_uri <http://purl.org/ontology/mo/meter> ?meter.
+    ?song_robust_uri <http://purl.org/ontology/mo/tempo> ?tempo.
+  }
+}
+ORDER BY ASC(?year) ?country
+
+
Index: main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/requirements.txt
===================================================================
--- main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/requirements.txt	(revision 35141)
+++ main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/requirements.txt	(revision 35141)
@@ -0,0 +1,12 @@
+certifi==2020.12.5
+chardet==4.0.0
+idna==2.10
+isodate==0.6.0
+musicbrainzngs==0.7.1
+pyparsing==2.4.7
+rdflib==5.0.0
+requests==2.25.1
+six==1.15.0
+SPARQLWrapper==1.8.5
+spotipy==2.16.1
+urllib3==1.26.3
