Changeset 34978

Timestamp:

2021-03-27T22:16:25+13:00 (3 years ago)

Author:

davidb

Message:

Processing of table changed so 1956 double-entries could be correctly processed

Location:

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories

Files:

• esc-wikipedia-download-and-process-votes.py (modified) (2 diffs)
• escwikipedia.py (added)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/esc-wikipedia-download-and-process-votes.py

@@ -1 +1 @@
 #!/usr/bin/env python
 
-from __future__ import print_function
+### TODO
+### Grab image for competition year out of infobox???
 
+import json
 import os
 import re
 
-import bs4
-import wikipedia
+import argparse
+
+import escwikipedia
 
 
-cache_dir="cache-wikipedia"
+def gs_fileset_country_in_year(country_year_name, country_year_rec, nul_output_dir_name):
+
+    # Looking to build data-structure (for output as JSON) in the form
+    # { "FileSet":
+    #   [
+    #     { "FileName": "France1991\.nul" }, 
+    #     { "Description":
+    #       {
+    #         "Metadata":
+    #           [
+    #             { "name": "Place", "content": 12 }, 
+    #             { "name": "Title",  "content": "...." },  
+    #             ...
+    #           ]
+    #       }
+    #     }
+    #   ]
+    # }
+
+    metadata_array = []
+    
+    country_year_id = re.sub(r'\s+', '', country_year_name) 
+
+    metadata_array.append({"name": "Identifier", "content": country_year_id})
+        
+    for metadata_key in country_year_rec.keys():
+        metadata_val = country_year_rec.get(metadata_key)
+        metadata_array.append({"name": metadata_key, "content": metadata_val})
 
 
-def read_text_file(input_filename):
-    f = open(input_filename, "r")
-    text = f.read()
-    f.close()
+    # id encodes country and year,
+    filename_id = country_year_id + "\\.nul"
 
-    return text
+    nul_filename = os.path.join(nul_output_dir_name,country_year_id+".nul");
 
-def write_text_file(output_filename, text):
-    f = open(output_filename, "w")
-    f.write(text)
-    f.close()
+    print("  Creating: " + nul_filename)
+        
+    with open(nul_filename, 'w') as outfile:
+        outfile.write("")
+    
+    fileset = {
+        "FileSet" : [            
+            { "FileName": filename_id, },
+            { "Description" : { "Metadata" : metadata_array } }
+        ]
+    }
+              
+    return fileset
 
-def html_tablerows_to_hashmap(table_rows):
 
-    table_header = table_rows[0]
 
-    header_to_vals = {}
-    headers = []
+def gs_directory_metadata(all_country_year_array_of_recs, json_output_filename):
+
+    # Express the grouped from-country voting data
+    # in the Greenstone JSON metadata format:
+    
+    # { "DirectoryMetadata":
+    #   [
+    #     { "FileSet":
+    #       [
+    #         { "FileName": "France1991\.nul" },
+    #         { "Description":
+    #           {
+    #             "Metadata":
+    #              [
+    #                { "name": "Place", "content": "12" }, 
+    #                  ...
+    #              ]
+    #           }
+    #         }
+    #       ]
+    #     }
+    #     ...
+    #    ]
+    #  }
+
+    nul_output_dir_name = os.path.dirname(json_output_filename)
+    
+    directory_metadata = []
+
+    for all_in_given_year_recs in all_country_year_array_of_recs:
         
-    header_cols = table_header.find_all("th");
-    for header in header_cols:
-        header_label = header.contents[0].strip()
-        if (header_label == "Language(s)"):
-            header_label = "Language"
-            
-        headers.append(header_label)
-        header_to_vals[header_label] = []
+        for country_year_name in all_in_given_year_recs.keys():
+            country_year_rec = all_in_given_year_recs.get(country_year_name)
 
-    print("  Headers = " + ",".join(header_to_vals.keys()))
-    
-    for y in range(1, len(table_rows)):
-        tds = table_rows[y].find_all("td");
-        for x in range(0,len(tds)):
-            val = tds[x]
-            header_label = headers[x]
-            header_to_vals[header_label].append(val)
-            
-    return header_to_vals
+            fileset = gs_fileset_country_in_year(country_year_name, country_year_rec, nul_output_dir_name)
+            directory_metadata.append(fileset)
+        
 
-def convert_cols_to_country_year_recs(header_to_vals,year):
+    greenstone_metadata_json = { "DirectoryMetadata": directory_metadata }
 
-    country_year_recs = {}
+    return greenstone_metadata_json
 
-    a_href_re = re.compile(r"^.*" + str(year) + r"$")
-    
-    for country_tag in header_to_vals.get("Country"):
-        # print("@@@@@ td = " + repr(country_tag));
-        
-        #country = country_tag.find("a",href=re.compile(r"^.*" + str(year) + r"$")).string
-        country = country_tag.find("a",href=a_href_re).string
-        # print("**** country = " + country)
-        country_year = country+str(year)
-        
-        country_year_recs[country_year] = {}
-                    
-    for key in header_to_vals.keys():
-        if (key == "Country"):
-            continue
 
-        vals = header_to_vals.get(key)
-        
-        for l in range(0,len(vals)):
-            country_tag = header_to_vals.get("Country")[l]
-            country_flag_img = country_tag.find("img")
-            #country = country_tag.find("a",href=re.compile(r"^.*" + str(year) + r"$")).string
-            country = country_tag.find("a",href=a_href_re).string
-            country_year = country+str(year)
-            
-            val = vals[l]
-            
-            if key == "Artist":
-                a_val = val.find("a")
-                if (a_val is not None):
-                    val = a_val
-            elif key == "Song":
-                a_val = val.find("a")
-                if (a_val is not None):
-                    val = a_val
-            elif key == "Language":
-                a_val = val.find("a")
-                if (a_val is not None):
-                    val = a_val
-            elif key == "Place":
-                span_val = val.find("span")
-                if (span_val is not None):
-                    val = span_val
 
-            for inner_val in val.contents:
-                if (inner_val.string and re.search("[^\s]",inner_val.string)):
-                    val = inner_val
-                    break
-                
-            val = val.string.strip()
+def save_greenstone_json_metadata(greenstone_metadata_json,json_output_filename):
 
-            #print("country = " + country);
-            #print("key = " + key);
-            #print("*** storing: " + country + "[" + key + "] = " + val)
-            
-            country_year_recs[country_year][key] = val
-
-    return country_year_recs
-
-def output_country_year_recs(country_year_recs):
-
-    for country_name in country_year_recs.keys():
-        country_rec = country_year_recs.get(country_name)
-
-        print("[" + country_name + "]")
-              
-        for metadata_key in country_rec.keys():
-            metadata_val = country_rec.get(metadata_key)
-            print("  " + metadata_key + " = " + repr(metadata_val))
-        
-def process_category_page(year):
-    esc_wiki_page = "Eurovision_Song_Contest_" + str(year)
-    esc_wiki_page_file = esc_wiki_page.replace(" ","_") + ".html"
-    esc_wiki_page_filename = os.path.join(cache_dir,esc_wiki_page_file)
-
-    esc_year_html = ""
-    if not(os.path.exists(esc_wiki_page_filename)):
-        print("Retrieving Wikipedia page '" + esc_wiki_page + "'")
-        esc_year_wp = wikipedia.page(esc_wiki_page,auto_suggest=False,preload=True,redirect=True)
-        esc_year_html = esc_year_wp.html()
-
-        print("  Saving page to cache")
-        write_text_file(esc_wiki_page_filename,esc_year_html)
-    else:
-        print("Retrieving " + esc_wiki_page_file + " from cache")
-        esc_year_html = read_text_file(esc_wiki_page_filename)
-
-        
-    esc_year_soup = bs4.BeautifulSoup(esc_year_html, 'html.parser')
-    
-    final_result_ids = [ 'Grand_final', 'Final', 'Results' ]
-
-    results_heading = None
-    for fr_id in final_result_ids:
-
-        if ((year == 1996) and (fr_id == "Final")):
-            continue
-            
-        results_text_span = esc_year_soup.find("span",id=fr_id)
-        if (results_text_span is not None):
-            print("  Found Final Results heading with id: " + fr_id);
-            results_heading = results_text_span.parent
-            # print("**** parent tag: " + results_heading.name);
-            break
-
-    # print (results_heading)
-
-    results_table = results_heading.findNext('table')
-    table_rows = results_table.find_all('tr');
-    print("  " + esc_wiki_page_file + ": number of rows in Results table = " + str(len(table_rows)))
-
-    header_to_vals = html_tablerows_to_hashmap(table_rows)
-
-    country_year_recs = convert_cols_to_country_year_recs(header_to_vals,year)
-
-    print()
-    print("==========")
-
-    output_country_year_recs(country_year_recs)
+    with open(json_output_filename, 'w') as outfile:
+        json.dump(greenstone_metadata_json, outfile, indent=2)
     
     
@@ -182 +119 @@
     # https://en.wikipedia.org/wiki/Category:Countries_in_the_Eurovision_Song_Contest_1957
 
-    if not(os.path.exists(cache_dir)):
-        print("Making cache directory: " + cache_dir)
-        os.mkdir(cache_dir)
-            
-    for year in range(1956, 2020):
-        process_category_page(year)
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--startyear', type=int, default=1956)
+    parser.add_argument('--endyear',   type=int, default=1999)
+    parser.add_argument('output-file.json', nargs="?", default="metadata-esc-year/metadata-esc.json");
+    
+    args = parser.parse_args()
 
+    start_year = getattr(args,'startyear')
+    end_year   = getattr(args,'endyear')
     
-#    for row in table_rows:
-#        print("*** " + repr(row));
+    json_output_filename = getattr(args,'output-file.json')
+
+
+    all_country_year_recs = []
         
-#    for sibling in results_heading.previous_siblings:
-#        print(repr(sibling))
+    for year in range(start_year, end_year+1):
+        country_year_recs = escwikipedia.process_category_page(year)
 
+        all_country_year_recs.append(country_year_recs)
+        
+    print()
+    print("Generating Greenstone JSON metadata for:")
+    greenstone_metadata_json = gs_directory_metadata(all_country_year_recs,json_output_filename)
+    
+    print()
+    print("Saving output as: " + json_output_filename)    
+    save_greenstone_json_metadata(greenstone_metadata_json,json_output_filename)
+    print()
+        
     
 
-
-    
-    #td_tag.findNext('td')
-
-    # results_table = results_heading.findNext('table')
-    # print(repr(results_table))
-
-
-#ny = wikipedia.page("New York")
-#ny.html()
-#soup = BeautifulSoup("<html>a web page</html>", 'html.parser')
-#id_soup.p['id']