Changeset 35016

Timestamp:

2021-04-04T20:24:02+12:00 (3 years ago)

Author:

davidb

Message:

Changes in response to testing

File:

• main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/esc-wikipedia-download-and-detect-missing-cat-entries.py (modified) (8 diffs)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/esc-wikipedia-download-and-detect-missing-cat-entries.py

@@ -5 +5 @@
 
 
+import json
 import os
 import re
@@ -48 +49 @@
 
 
-def gs_directory_metadata(all_country_year_array_of_recs, json_output_filename):
+def gs_directory_metadata(missing_year_countries, json_output_filename):
 
 
@@ -55 +56 @@
     directory_metadata = []
 
-    for all_in_given_year_recs in all_country_year_array_of_recs:
-        
-        for country_year_name in all_in_given_year_recs.keys():
-            country_year_rec = all_in_given_year_recs.get(country_year_name)
-
+    for year in missing_year_countries.keys():
+        year_str = str(year)
+        
+        for country in missing_year_countries[year]:
+
+            country_year_name = country + year_str
+            country_year_rec = missing_year_countries[year][country]
+            
+            # dig out metadata rec based on year and country
+            
             fileset = gs_fileset_country_in_year(country_year_name, country_year_rec, nul_output_dir_name)
             directory_metadata.append(fileset)
@@ -70 +76 @@
 
 
-#def save_greenstone_json_metadata(greenstone_metadata_json,json_output_filename):
-#
-#    with open(json_output_filename, 'w') as outfile:
-#        json.dump(greenstone_metadata_json, outfile, indent=2)
-    
+def save_greenstone_json_metadata(greenstone_metadata_json,json_output_filename):
+
+    with open(json_output_filename, 'w') as outfile:
+        json.dump(greenstone_metadata_json, outfile, indent=2)
+    
+
+def generate_sparql_query(sparql_values_output_filename,missing_year_countries):
+
+    sparql_values_template_filename = sparql_values_output_filename + ".in"
+
+    missing_uri_lines = []
+    
+    for year in missing_year_countries.keys():
+        year_str = str(year)
+        for country in missing_year_countries[year]:
+            # Example:
+            #  dbr:Cyprus_in_the_Eurovision_Song_Contest_2010
+            
+            country_wikisafe = country.replace(" ","_")
+            
+            missing_uri = "      (dbr:" + country_wikisafe + "_in_the_Eurovision_Song_Contest_" + year_str+")"
+            
+            missing_uri_lines.append(missing_uri)
+            
+            
+    missing_uri_lines_text = "\n".join(missing_uri_lines)
+    util.write_text_file_from_template(sparql_values_template_filename,"**missing-country-year-uris**",missing_uri_lines_text,sparql_values_output_filename)
+
+#    problem_category_in_year_filename = "../problem-lod-lists/dbpedia-problem-category-in-year.sparql"    
+#    shutil.copyfile(sparql_values_output_filename,problem_category_in_year_filename)
+                    
+
+
     
 if __name__ == "__main__":
@@ -100 +134 @@
     parser.add_argument('--endyear',   type=int, default=2019)
     parser.add_argument('--cachedir', default="cache-wikipedia")
-    parser.add_argument('output-file.sparql', nargs="?", default="dbpedia--countries-missing-from-esc-category-in-the-year.sparql");
+    parser.add_argument('--queryfile.sparql', default=None)
+    parser.add_argument('output-file.json', nargs="?", default="metadata.json");
     
     args = parser.parse_args()
@@ -110 +145 @@
     escwikipedia.init_cache(cache_dir)
     
-    sparql_values_output_filename  = getattr(args,'output-file.sparql')
-    sparql_values_template_filename = sparql_values_output_filename + ".in"
+    sparql_values_output_filename  = getattr(args,'queryfile.sparql')
+
+    json_output_filename  = getattr(args,'output-file.json')
+
 
     final_result_ids = [ 'Grand_final', 'Final', 'Results' ]
@@ -128 +165 @@
             article_semifinal_country_year_recs = escwikipedia.process_article_page(article_country_year_html,year,semifinal_pair_result_ids, stop_at_first=False)
         article_country_year_recs = util.merge_dicts(article_final_country_year_recs,article_semifinal_country_year_recs)
+
         article_countries = {}
         for country_year in article_country_year_recs.keys():
             article_country = re.search(r"(^.*)\d{4}(?:\w+)?",country_year).group(1)
             
-            article_countries[article_country] = 1
+            article_countries[article_country] = article_country_year_recs[country_year]
                 
         category_countries = escwikipedia.process_category_page(year)
@@ -162 +200 @@
     print()
 
-    missing_uri_lines = []
-    
-    for year in missing_year_countries.keys():
-        year_str = str(year)
-        for country in missing_year_countries[year]:
-            # Example:
-            #  dbr:Cyprus_in_the_Eurovision_Song_Contest_2010
-            
-            country_wikisafe = country.replace(" ","_")
-            
-            missing_uri = "      (dbr:" + country_wikisafe + "_in_the_Eurovision_Song_Contest_" + year_str+")"
-            
-            missing_uri_lines.append(missing_uri)
-            
-            #print("  " + missing_uri)
-
-    missing_uri_lines_text = "\n".join(missing_uri_lines)
-    util.write_text_file_from_template(sparql_values_template_filename,"**missing-country-year-uris**",missing_uri_lines_text,sparql_values_output_filename)
-
-#    problem_category_in_year_filename = "../problem-lod-lists/dbpedia-problem-category-in-year.sparql"    
-#    shutil.copyfile(sparql_values_output_filename,problem_category_in_year_filename)
-                    
-    print()
-
-
-        
-    
-
+    greenstone_metadata_json = gs_directory_metadata(missing_year_countries,json_output_filename)
+    save_greenstone_json_metadata(greenstone_metadata_json,json_output_filename)
+    
+    if (sparql_values_output_filename is not None):
+        generate_sparql_query(sparql_values_output_filename,missing_year_countries)    
+        print()