Changeset 34976 for main

Timestamp:

2021-03-26T17:46:12+13:00 (3 years ago)

Author:

davidb

Message:

Output method added in that prints (for now) the metadata for country+year it is building up

File:

• main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/esc-wikipedia-download-and-process-votes.py (modified) (6 diffs)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/esc-wikipedia-download-and-process-votes.py

@@ -4 +4 @@
 
 import os
+import re
 
 import bs4
@@ -27 +28 @@
 
     table_header = table_rows[0]
-    # print("table header = " + repr(table_header))
 
     header_to_vals = {}
@@ -38 +38 @@
             header_label = "Language"
             
-        # print("header: '" + header_label+"'")
-
         headers.append(header_label)
         header_to_vals[header_label] = []
@@ -54 +52 @@
     return header_to_vals
 
-def convert_cols_to_country_recs(header_to_vals):
+def convert_cols_to_country_recs(header_to_vals,year):
 
     country_recs = {}
 
-    for country in header_to_vals.get("Country"):
+    for country_tag in header_to_vals.get("Country"):
+        # print("@@@@@ td = " + repr(country_tag));
+        
+        country = country_tag.find("a",href=re.compile(r"^.*" + str(year) + r"$")).string
+        # print("**** country = " + country)
+        
         country_recs[country] = {}
                     
@@ -68 +71 @@
         
         for l in range(0,len(vals)):
-            country = header_to_vals.get("Country")[l]
+            country_tag = header_to_vals.get("Country")[l]
+            country_flag_img = country_tag.find("img")
+            country = country_tag.find("a",href=re.compile(r"^.*" + str(year) + r"$")).string
+            
             val = vals[l]
-
+            print("@@@ val = " + repr(val))
+            
+            if key == "Artist":
+                a_val = val.find("a")
+                if (a_val is not None):
+                    val = a_val
+            elif key == "Song":
+                a_val = val.find("a")
+                if (a_val is not None):
+                    val = a_val
+            elif key == "Language":
+                a_val = val.find("a")
+                if (a_val is not None):
+                    val = a_val
+            elif key == "Place":
+                span_val = val.find("span")
+                if (span_val is not None):
+                    val = span_val
+
+            for inner_val in val.contents:
+                if (inner_val.string and re.search("[^\s]",inner_val.string)):
+                    val = inner_val
+                    break
+                
+            val = val.string.strip()
+
+            #print("country = " + country);
+            #print("key = " + key);
+            #print("*** storing: " + country + "[" + key + "] = " + val)
+            
             country_recs[country][key] = val
 
     return country_recs
 
+def output_country_recs(country_recs):
+
+    for country_name in country_recs.keys():
+        country_rec = country_recs.get(country_name)
+
+        print("[" + country_name + "]")
+              
+        for metadata_key in country_rec.keys():
+            metadata_val = country_rec.get(metadata_key)
+            print("  " + metadata_key + " = " + repr(metadata_val))
+        
 def process_category_page(year):
     esc_wiki_page = "Eurovision_Song_Contest_" + str(year)
@@ -118 +164 @@
     header_to_vals = html_tablerows_to_hashmap(table_rows)
 
-    convert_cols_to_country_recs(header_to_vals)
-    
+    country_recs = convert_cols_to_country_recs(header_to_vals,year)
+
+    print()
+    print("==========")
+
+    output_country_recs(country_recs)