Changeset 35032 for main

Timestamp:

2021-04-05T15:48:36+12:00 (3 years ago)

Author:

davidb

Message:

optional extra_metadata argument added in

Location:

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories

Files:

• esc-wikipedia-download-and-detect-missing-cat-entries.py (modified) (1 diff)
• esc-wikipedia-download-and-process-votes.py (modified) (1 diff)
• escwikipedia.py (modified) (5 diffs)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/esc-wikipedia-download-and-detect-missing-cat-entries.py

@@ -197 +197 @@
 
     greenstone_metadata_json = gs_directory_metadata(missing_year_countries,json_output_filename)
+    print("Saving output as: " + json_output_filename)
     save_greenstone_json_metadata(greenstone_metadata_json,json_output_filename)
     

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/esc-wikipedia-download-and-process-votes.py

@@ -150 +150 @@
         article_year_html = escwikipedia.retrieve_article_page(year)
         country_year_recs = escwikipedia.process_article_page(article_year_html,year,final_result_ids,
-                                                                      stop_at_first=True)
+                                                              stop_at_first=True,
+                                                              extra_metadata={"Final": True})
+
 
         all_country_year_recs.append(country_year_recs)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/errata-categories/escwikipedia.py

@@ -136 +136 @@
 
 
-def convert_cols_to_country_year_recs(header_to_vals,year,accumulated_country_year_recs):
+def convert_cols_to_country_year_recs(header_to_vals,year,accumulated_country_year_recs,extra_metadata):
 
     a_href_re = re.compile(r"^.*" + str(year) + r"$")
@@ -158 +158 @@
         country_year = country + str(year)
 
-        this_country_year_rec = { "Country": country, "Year": year }
+        this_country_year_rec = {}
+        if (extra_metadata is not None):
+            this_country_year_rec = extra_metadata.copy()
+            
+        this_country_year_rec["Country"] = country
+        this_country_year_rec["Year"]    = year 
         
         for key in header_to_vals.keys():
             if (key == "Country"):
-                country_flag_img = country_tag.find("img")
-                #print(repr(country_flag_img))
-                #flag_img = country_flag_img.src
-                #if (flag_img is None):
-                #    print("***** Flag image empty!!!!")
-                    
+                country_flag_img = country_tag.find("img")                    
                 this_country_year_rec["FlagImg"] = str(country_flag_img)
                 continue
@@ -189 +189 @@
                 if (span_val is not None):
                     val = span_val
+            elif key == "Points":
+                key = "VoteGrandTotal"
 
             for inner_val in val.contents:
@@ -259 +261 @@
     return esc_article_year_html
 
-def process_article_page(esc_article_year_html,year,result_ids, stop_at_first):
+def process_article_page(esc_article_year_html,year,result_ids, stop_at_first, extra_metadata=None):
 
     country_year_recs = {}
@@ -294 +296 @@
         header_to_vals = html_tablerows_to_hashmap(table_rows)
 
-        convert_cols_to_country_year_recs(header_to_vals,year,country_year_recs)
+        convert_cols_to_country_year_recs(header_to_vals,year,country_year_recs, extra_metadata)
 
     print("==========")