Changeset 34990

Timestamp:

2021-04-04T13:07:02+12:00 (3 years ago)

Author:

davidb

Message:

Refactoring in response to developing top-level PREPARE script

Location:

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories

Files:

• esc-wikipedia-download-and-detect-missing-cat-entries.py (modified) (5 diffs)
• esc-wikipedia-download-and-process-votes.py (modified) (3 diffs)
• escwikipedia.py (modified) (4 diffs)
• util.py (modified) (1 diff)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/esc-wikipedia-download-and-detect-missing-cat-entries.py

@@ -7 +7 @@
 import os
 import re
+# import shutil
 
 import argparse
@@ -98 +99 @@
     parser.add_argument('--startyear', type=int, default=1956)
     parser.add_argument('--endyear',   type=int, default=2019)
-    # parser.add_argument('output-file.json', nargs="?", default="metadata-esc-year/metadata-esc.json");
+    parser.add_argument('--cachedir', default="cache-wikipedia")
+    parser.add_argument('output-file.sparql', nargs="?", default="dbpedia--countries-missing-from-esc-category-in-the-year.sparql");
     
     args = parser.parse_args()
@@ -104 +106 @@
     start_year = getattr(args,'startyear')
     end_year   = getattr(args,'endyear')
+
+    cache_dir = getattr(args,'cachedir')
+    escwikipedia.init_cache(cache_dir)
     
-    # json_output_filename = getattr(args,'output-file.json')
+    sparql_values_output_filename  = getattr(args,'output-file.sparql')
+    sparql_values_template_filename = sparql_values_output_filename + ".in"
+
     final_result_ids = [ 'Grand_final', 'Final', 'Results' ]
     semifinal_result_ids = [ 'Semi-final']
     semifinal_pair_result_ids = [ 'Semi-final_1', 'Semi-final_2' ]
+
+    missing_year_countries = {}
     
     for year in range(start_year, end_year+1):
@@ -135 +144 @@
                 del article_countries[cat_country]
             else:
-                print("Country listed in Category page that did not reach Fina/Semifinal: " + cat_country)
+                print("Country listed in Category page that did not reach Final/Semifinal: " + cat_country)
                 
         # Missing countries are the ones left in 'article_countries'
@@ -146 +155 @@
             print("++++++++++")
 
+            missing_year_countries[year] = article_countries
+
         print("==========")
         print()
-        
+
+    print()
+
+    missing_uri_lines = []
+    
+    for year in missing_year_countries.keys():
+        year_str = str(year)
+        for country in missing_year_countries[year]:
+            # Example:
+            #  dbr:Cyprus_in_the_Eurovision_Song_Contest_2010
+            
+            country_wikisafe = country.replace(" ","_")
+            
+            missing_uri = "      (dbr:" + country_wikisafe + "_in_the_Eurovision_Song_Contest_" + year_str+")"
+            
+            missing_uri_lines.append(missing_uri)
+            
+            #print("  " + missing_uri)
+
+    missing_uri_lines_text = "\n".join(missing_uri_lines)
+    util.write_text_file_from_template(sparql_values_template_filename,"**missing-country-year-uris**",missing_uri_lines_text,sparql_values_output_filename)
+
+#    problem_category_in_year_filename = "../problem-lod-lists/dbpedia-problem-category-in-year.sparql"    
+#    shutil.copyfile(sparql_values_output_filename,problem_category_in_year_filename)
+                    
+    print()
+
 
         

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/esc-wikipedia-download-and-process-votes.py

@@ -89 +89 @@
     #  }
 
+    print("**** json_output_filename = " + json_output_filename)
+    
     nul_output_dir_name = os.path.dirname(json_output_filename)
     
@@ -122 +124 @@
     parser.add_argument('--startyear', type=int, default=1956)
     parser.add_argument('--endyear',   type=int, default=2019)
+    parser.add_argument('--cachedir', default="cache-wikipedia")
     parser.add_argument('output-file.json', nargs="?", default="metadata-esc-year/metadata-esc.json");
     
@@ -128 +131 @@
     start_year = getattr(args,'startyear')
     end_year   = getattr(args,'endyear')
-    
+
+    cache_dir = getattr(args,'cachedir')
+    escwikipedia.init_cache(cache_dir)
+
     json_output_filename = getattr(args,'output-file.json')
 

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/escwikipedia.py

@@ -18 +18 @@
 import wikipedia
 
-import wikipediaapi
-
 DEBUG=False
 
 
-cache_dir="cache-wikipedia"
-
-# Global Init
-if not(os.path.exists(cache_dir)):
-    print("Making cache directory: " + cache_dir)
-    os.mkdir(cache_dir)
+cache_dir_=None
+
+def init_cache(cache_dir):
+    global cache_dir_
+    cache_dir_ = cache_dir
+    if not(os.path.exists(cache_dir_)):
+        print("Making cache directory: " + cache_dir_)
+        os.mkdir(cache_dir_)
     
 
@@ -241 +241 @@
     esc_wiki_page = "Eurovision_Song_Contest_" + str(year)
     esc_wiki_page_file = esc_wiki_page.replace(" ","_") + ".html"
-    esc_wiki_page_filename = os.path.join(cache_dir,esc_wiki_page_file)
+    esc_wiki_page_filename = os.path.join(cache_dir_,esc_wiki_page_file)
 
     esc_article_year_html = ""
@@ -308 +308 @@
     esc_wiki_page = "Category:Countries_in_the_Eurovision_Song_Contest_" + str(year)
     esc_wiki_page_file = esc_wiki_page.replace(" ","_") + ".html"
-    esc_wiki_page_filename = os.path.join(cache_dir,esc_wiki_page_file)
+    esc_wiki_page_filename = os.path.join(cache_dir_,esc_wiki_page_file)
 
     esc_cat_year_html = ""
@@ -345 +345 @@
 
 
-def cat_test():
-
-    wiki_wiki = wikipediaapi.Wikipedia("en",extract_format=wikipediaapi.ExtractFormat.HTML)
-
-    cat_title = "Category:Countries_in_the_Eurovision_Song_Contest_1956"
-    cat = wiki_wiki.page(cat_title)
-
-    print("**** html = " + cat.text)
-    
-    
-    print("Category members: " + cat_title)
-    for p in cat.categorymembers.values():
-        if p.namespace == wikipediaapi.Namespace.CATEGORY:
-            # it is category, so you have to make decision
-            # if you want to fetch also text from pages that belong
-            # to this category
-            print("CatNS")
-            print(p)
-        elif p.namespace == wikipediaapi.Namespace.MAIN:
-            # it is page => we can get text
-            print("MAIN")
-            print(p)
-            print(p.text)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/util.py

@@ -7 +7 @@
     z.update(y)    # modifies z with y's keys and values & returns None
     return z
+
+
+def write_text_file_from_template(input_template_filename,marker,text,output_filename):
+
+    fin = open(input_template_filename, "r")
+    lines = fin.readlines()
+    fin.close()
+
+    fout = open(output_filename, "w")
+    for line in lines:
+        line = line.replace(marker,text)
+        fout.write(line)
+
+    fout.close()