Changeset 34975 for main

Timestamp:

2021-03-25T18:47:20+13:00 (3 years ago)

Author:

davidb

Message:

Digging into the country results table

File:

• main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/esc-wikipedia-download-and-process-votes.py (modified) (3 diffs)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/esc-wikipedia-download-and-process-votes.py

@@ -24 +24 @@
     f.close()
 
+def html_tablerows_to_hashmap(table_rows):
+
+    table_header = table_rows[0]
+    # print("table header = " + repr(table_header))
+
+    header_to_vals = {}
+    headers = []
+        
+    header_cols = table_header.find_all("th");
+    for header in header_cols:
+        header_label = header.contents[0].strip()
+        if (header_label == "Language(s)"):
+            header_label = "Language"
+            
+        # print("header: '" + header_label+"'")
+
+        headers.append(header_label)
+        header_to_vals[header_label] = []
+
+    print("  Headers = " + ",".join(header_to_vals.keys()))
     
+    for y in range(1, len(table_rows)):
+        tds = table_rows[y].find_all("td");
+        for x in range(0,len(tds)):
+            val = tds[x]
+            header_label = headers[x]
+            header_to_vals[header_label].append(val)
+            
+    return header_to_vals
+
+def convert_cols_to_country_recs(header_to_vals):
+
+    country_recs = {}
+
+    for country in header_to_vals.get("Country"):
+        country_recs[country] = {}
+                    
+    for key in header_to_vals.keys():
+        if (key == "Country"):
+            continue
+
+        vals = header_to_vals.get(key)
+        
+        for l in range(0,len(vals)):
+            country = header_to_vals.get("Country")[l]
+            val = vals[l]
+
+            country_recs[country][key] = val
+
+    return country_recs
 
 def process_category_page(year):
@@ -50 +99 @@
     results_heading = None
     for fr_id in final_result_ids:
-        
+
+        if ((year == 1996) and (fr_id == "Final")):
+            continue
+            
         results_text_span = esc_year_soup.find("span",id=fr_id)
         if (results_text_span is not None):
             print("  Found Final Results heading with id: " + fr_id);
-            results_heading = results_text_span.parent        
+            results_heading = results_text_span.parent
+            # print("**** parent tag: " + results_heading.name);
             break
 
@@ -61 +114 @@
     results_table = results_heading.findNext('table')
     table_rows = results_table.find_all('tr');
+    print("  " + esc_wiki_page_file + ": number of rows in Results table = " + str(len(table_rows)))
 
-    print(esc_wiki_page_file + ": number of rows in Results table = " + str(len(table_rows)))
+    header_to_vals = html_tablerows_to_hashmap(table_rows)
 
+    convert_cols_to_country_recs(header_to_vals)
+    
+    
     
 if __name__ == "__main__":