Changeset 35862 for main/trunk

Timestamp:

2022-01-03T15:45:40+13:00 (2 years ago)

Author:

davidb

Message:

Extended to support startyear and endyear

File:

• main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/eurosparqlify.py (modified) (5 diffs)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/spotify-musicbrainz/eurosparqlify.py

@@ -47 +47 @@
     return "http://motools.sourceforge.net/keys/keys.owl#" + keys[key] + modes[mode]
 
-def get_dbp_results():
+def get_dbp_results(start_year,end_year):
     euroSparql = SPARQLWrapper(euroEndpoint)
     euroSparql.setReturnFormat(JSON)
@@ -53 +53 @@
     dbpSparql.setReturnFormat(JSON)
 
+    opt_filter_terms = []
+    if (start_year is not None):
+        opt_filter_terms.append("      FILTER ( <http://www.w3.org/2001/XMLSchema#integer>(?entryYear) >= " + str(start_year) + ") .");
+
+    if (end_year is not None):
+        opt_filter_terms.append("      FILTER ( <http://www.w3.org/2001/XMLSchema#integer>(?entryYear) <= " + str(end_year) + ") .");
+
+    opt_filter = ""
+
+    if (len(opt_filter_terms) > 0):
+        opt_filter_terms.insert(0,"bind( REPLACE(str(?entryUri), \".*(\\\\d{4})\", \"$1\") AS ?entryYear).")
+            
+        opt_filter = "\n".join(opt_filter_terms)
+            
+        # bind( REPLACE(str(?entryUri), ".*(\\d{4})", "$1") AS ?entryYear).
+
+        # prefix xsd:    <http://www.w3.org/2001/XMLSchema#>
+            #    FILTER ( http://www.w3.org/2001/XMLSchema\#integer(?entryYear) >= 2014).
+            # FILTER ( http://www.w3.org/2001/XMLSchema\#integer(?entryYear) <= 2016).
+
+    
+
     # how many entry - song pairings are there?
-    euroSparql.setQuery("""
-      SELECT (COUNT(DISTINCT *) as ?count) WHERE {
-        GRAPH <https://so-we-must-think.space/greenstone3/eurovision-library/collection/eurovision> {
+    count_query_str = """
+      SELECT (COUNT(DISTINCT *) as ?count) WHERE {{
+        GRAPH <https://so-we-must-think.space/greenstone3/eurovision-library/collection/eurovision> {{
           ?entryUri <http://greenstone.org/gsdlextracted#song_robust_uri> ?songUri .
-        }
-      } 
-    """)
+          {0}
+        }}
+      }}
+    """.format(opt_filter)
+
+    
+    euroSparql.setQuery(count_query_str)
+    
     countResult = euroSparql.query().convert()
     count = int(countResult["results"]["bindings"][0]["count"]["value"])
 
+    print("Number of matching entries: " + str(count))
+    sleep(3)
+    
     # retrieve entry-song pairings and corresponding dbpedia matches IN CHUNKS OF 500
     n=0
     dbpBindings = []
     while n < count:
-        euroSparql.setQuery("""
-          SELECT DISTINCT ?entryUri ?songUri WHERE {
-            graph <https://so-we-must-think.space/greenstone3/eurovision-library/collection/eurovision> {
+        query_offset_n_str = """
+          SELECT DISTINCT ?entryUri ?songUri WHERE {{
+            graph <https://so-we-must-think.space/greenstone3/eurovision-library/collection/eurovision> {{
               ?entryUri <http://greenstone.org/gsdlextracted#song_robust_uri> ?songUri .
-            }
-          } LIMIT 500 OFFSET """ + str(n) + """
-        """)
+              {0}
+            }}
+          }} LIMIT 500 OFFSET {1}
+        """.format(opt_filter,str(n))
+        
+        euroSparql.setQuery(query_offset_n_str)
 
         print("Issuing Greenstone SPARQL query with n=", n)
@@ -216 +249 @@
 
     parser = argparse.ArgumentParser()
+
+    parser.add_argument('-sy', '--startyear', type=int, help="Limit entries to be from this year and onwards")
+    parser.add_argument('-ey', '--endyear',   type=int, help="Limit entries to be up to this year")
+
     parser.add_argument('-mb', '--musicbrainz',   action='store_true', help="Query the MusicBrainz API")
     parser.add_argument('-aa', '--audioanalysis', action='store_true', help="Include audio analysis features in output data")
@@ -224 +261 @@
     args = parser.parse_args()
 
+    start_year = getattr(args,'startyear');
+    end_year   = getattr(args,'endyear');
+        
     query_musicbrainz     = getattr(args,'musicbrainz');
     output_audio_analysis = getattr(args,'audioanalysis');
@@ -232 +272 @@
 
     
-    dbp_results = get_dbp_results();
+    dbp_results = get_dbp_results(start_year,end_year);
     outcomes = process_dbp_results(dbp_results, query_musicbrainz, output_audio_analysis)
     metadata_records = outcomes[0]