Changeset 34960 for main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/voting-excel/xlsx-tocountry-jsonmetadata.py

Timestamp:

2021-03-09T23:30:43+13:00 (3 years ago)

Author:

davidb

Message:

Changes associated with adding in Combined and Single Voting totals; adjustments to DL based on Terhi's feedback

File:

• main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/voting-excel/xlsx-tocountry-jsonmetadata.py (modified) (10 diffs)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/voting-excel/xlsx-tocountry-jsonmetadata.py

@@ -17 +17 @@
 jury_from_countries = {}
 tele_from_countries = {}
-
-
+comb_from_countries = {}
+sing_from_countries = {}
+
+# 1997:
+# Televoting by
+#  Austria, Germany, Sweeden, Switzerland, United Kingdom
+
+# 1998-2000: Televote, with Jury voting as backup plan
+
+# 2001-2002: All televote, or 50-50 mix if country so chooses, Jury backup
+#    2001: Jury: Bosnia & Herzegovina, Turkey, Russia
+#    2001: Mix:  Croatia, Greece, Malta
+#
+#    2002: Jury: Bosnia & Herzegovina, Turkey, Russia, FYR Macedonia, Romania
+#    2002: Mix:  Cyprus, Greece, Spain, Croatia, Finland, Malta, Slovenia, Lithuania
+
+# 2003: back to all Televote, with Jury voting as backup
+# 2004-2008: same as 2003
+
+# 2009-2012: Combined, except San Marino which was 100% Jury (due to size), Jury as backup
+
+# 2013-2015: As before, but how two types of votes combined different
+
+# 2016-      Jury and Tele given separately, added in
+
+def XXXXvoting_mode(year):
+
+    vote_mode = None
+
+    if (year>=2013):
+        # Separate voting: Jury and Tele 
+        vote_mode = "Separated"
+    elif ((year>=2001) and (year<=2012)):
+        vote_mode = "Combined"
+    else:
+        vote_mode = "Single"
+
+    return vote_mode
+
+
+def XXXXnuanced_voting_type(country,year):
+
+    vote_type = None
+
+    if (year>=2013):
+        # Separate voting: Jury and Tele 
+        vote_type = "JT"
+        
+    elif ((year>=2009) and (year<=2012)):
+        if (country=="San Marino"):
+            vote_type = "J"
+        else:
+            # Combined Jury and Tele
+            vote_type = "C"
+    elif ((year>=2003) and (year<=2008)):
+        vote_type = "T"
+    elif (year==2002):
+        jury_list = [ "Bosnia & Herzegovina", "Turkey", "Russia", "FYR Macedonia", "Romania"]
+        comb_list = [ "Cyprus", "Greece", "Spain", "Croatia", "Finland", "Malta", "Slovenia", "Lithuania" ]
+        if (country in jury_list):
+            vote_type = "J"
+        elif (country in comb_list):
+            vote_type = "C"
+        else:
+            vote_type = "T"
+    elif (year==2001):
+        jury_list = [ "Bosnia & Herzegovina", "Turkey", "Russia" ]
+        comb_list = [ "Croatia", "Greece", "Malta" ]
+        if (country in jury_list):
+            vote_type = "J"
+        elif (country in comb_list):
+            vote_type = "C"
+    elif ((year>=1998) and (year<=2000)):
+        vote_type = "T"
+    elif (year==1997):
+        tele_list = [ "Austria", "Germany", "Sweeden", "Switzerland", "United Kingdom" ]
+        if (country in tele_list):
+            vote_type = "T"
+        else:
+            vote_type = "J"
+    else:
+        vote_type = "J"
+
+    return vote_type
+
+
+            
 def create_to_country_voting_groups(data_hashmap_array):
 
@@ -68 +153 @@
 
 
-def fileset_voting_for_esc_country_in_year(to_country_year_votes,countrylabel_to_uri_mapping):
+def create_to_year_then_country_voting_groups(data_hashmap_array):
+
+    # Example values for header-names
+    #   (semi-) final:      f, sf
+    #   Jury or Televoting: J, T
+    #   Year:               1975, ...
+    #   To country:         Belgium
+    #   From country:       Belgium
+    #   Edition:            1975f, 1975sf
+
+    # Build array of year groups
+    #   A year group contains within it a country grouping that in turn
+    #   includes all the votes that country receive that year
+    year_groups = []
+
+    # Rely on sorting as done in first line of create_to_country_voting_groups,
+    # which is the (innermost) place where the sorting is needed
+    country_groups = create_to_country_voting_groups(data_hashmap_array)
+    
+    prev_to_country_votes = country_groups[0]
+    year_group = [ ]
+
+    i = 1
+    num_rows = len(country_groups)
+
+    while (i < num_rows):        
+        year_group.append(prev_to_country_votes)
+        to_country_votes = country_groups[i]
+
+        if (to_country_votes[0].get('Year') != prev_to_country_votes[0].get('Year')):
+            # moving on to a new year group
+##            print("**** year change from:" + str(to_country_votes[0].get('Year')) + " to " + str(prev_to_country_votes[0].get('Year')))
+            year_groups.append(year_group)
+            year_group = [ ]
+            
+        prev_to_country_votes = to_country_votes
+        i = i + 1
+        
+    year_group.append(prev_to_country_votes)        
+    year_groups.append(year_group)
+
+    print("****")
+    print("Number of years voting is available for: " + str(len(year_groups)));
+    print("****")
+    
+    return year_groups
+
+
+
+def augment_voting_for_esc_country_in_year(to_country_year_votes,countrylabel_to_uri_mapping):
+
+    jury_metadata_vals = []
+    tele_metadata_vals = []
+    comb_metadata_vals = []
+    sing_metadata_vals = []
+
+    to_country_jury_total = 0
+    to_country_tele_total = 0
+    to_country_comb_total = 0
+    to_country_sing_total = 0
+
+    to_country_grand_total = 0
+    
+    # Use the first record to be a representative for 'top level' (tl)
+    # metadata about the votes cast 'To country'
+    tl_rec = to_country_year_votes[0]
+
+    tl_to_country = tl_rec.get('To country')
+    tl_year       = tl_rec.get('Year')
+    tl_final      = tl_rec.get("(semi-) final") == "f"    
+
+    tl_to_country_id = re.sub(r'\s+', '', tl_to_country) + str(tl_year)
+
+    tl_rec["Final"] = tl_final
+
+    # tl_to_country_lc = tl_to_country.lower()
+    tl_to_country_uri = countrylabel_to_uri_mapping.get(tl_to_country)
+    if (tl_to_country_uri != None):
+        tl_rec["CountryDBURI"] = tl_to_country_uri
+    else:
+        util.eprint("Warning: Failed to find DBpedia Country URI match for: " + tl_to_country)
+                
+    for to_country_year_vote in to_country_year_votes:
+#        to_country   = to_country_year_vote.get('To country')
+        year         = to_country_year_vote.get('Year')
+        from_country = to_country_year_vote.get('From country')
+        vote_type    = to_country_year_vote.get('Jury or Televoting')
+        points       = to_country_year_vote.get('Points')
+
+        id_from_country = re.sub(r'\s+', '', from_country)
+            
+#        tl_rec[id_from_country+"-"+vote_type] = points
+
+        vote_mode = xlsxutil.voting_mode(year)
+        nuanced_vote_type = xlsxutil.nuanced_voting_type(from_country,year)
+
+        if (vote_mode == "Separated"):
+           
+            if (vote_type == "J"):
+                jury_metadata_vals.append(id_from_country+"-J")
+                jury_from_countries[id_from_country] = 1
+                to_country_jury_total = to_country_jury_total + points
+            
+            elif (vote_type == "T"):
+                tele_metadata_vals.append(id_from_country+"-T")
+                tele_from_countries[id_from_country] = 1
+                to_country_tele_total = to_country_tele_total + points
+            else:            
+                util.eprint("Warning: Unrecognized voting type: " + vote_type)
+
+        elif (vote_mode == "Combined"):
+            comb_metadata_vals.append(id_from_country+"-C")
+            comb_from_countries[id_from_country] = 1
+            to_country_comb_total = to_country_comb_total + points
+        else:
+            # Single
+            sing_metadata_vals.append(id_from_country+"-S")
+            sing_from_countries[id_from_country] = 1
+            to_country_sing_total = to_country_sing_total + points
+
+        to_country_grand_total = to_country_grand_total + points
+            
+    tl_rec["VoteMode"] = vote_mode
+        
+    if (len(jury_metadata_vals)>0):
+        tl_rec["JuryVotesJSON"] = jury_metadata_vals
+        tl_rec["JuryVotesTotal"] = to_country_jury_total
+    if (len(tele_metadata_vals)>0):
+        tl_rec["TeleVotesJSON"] = tele_metadata_vals
+        tl_rec["TeleVotesTotal"] = to_country_tele_total
+    if (len(comb_metadata_vals)>0):
+        tl_rec["CombVotesJSON"] = comb_metadata_vals
+        tl_rec["CombVotesTotal"] = to_country_comb_total
+    if (len(sing_metadata_vals)>0):
+        tl_rec["SingVotesJSON"] = sing_metadata_vals
+        tl_rec["SingVotesTotal"] = to_country_sing_total
+
+    tl_rec["VoteGrandTotal"] = to_country_grand_total
+
+def fileset_voting_for_esc_country_in_year(to_country_year_votes):
 
     # Looking to build data-structure (for output as JSON) in the form
@@ -89 +313 @@
     metadata_array = []
 
-    jury_metadata_vals = []
-    tele_metadata_vals = []
-
-    to_country_jury_total = 0
-    to_country_tele_total = 0
+#    jury_metadata_vals = []
+#    tele_metadata_vals = []
+
+#    to_country_jury_total = 0
+#    to_country_tele_total = 0
 
     # Use the first record to be a representative for 'top level' (tl)
@@ -101 +325 @@
     tl_to_country = tl_rec.get('To country')
     tl_year       = tl_rec.get('Year')
-    tl_final      = tl_rec.get("(semi-) final") == "f"    
+    # tl_final      = tl_rec.get("(semi-) final") == "f"    
 
     tl_to_country_id = re.sub(r'\s+', '', tl_to_country) + str(tl_year)
 
+    tl_final = tl_rec["Final"]
+    tl_to_country_uri = tl_rec.get("CountryDBURI") 
+
+    vote_mode = tl_rec.get("VoteMode")
+    
+    jury_metadata_vals = tl_rec.get("JuryVotesJSON") 
+    tele_metadata_vals = tl_rec.get("TeleVotesJSON") 
+    comb_metadata_vals = tl_rec.get("CombVotesJSON") 
+    sing_metadata_vals = tl_rec.get("SingVotesJSON") 
+
+    to_country_jury_total = tl_rec.get("JuryVotesTotal") 
+    to_country_tele_total = tl_rec.get("TeleVotesTotal") 
+    to_country_comb_total = tl_rec.get("CombVotesTotal") 
+    to_country_sing_total = tl_rec.get("SingVotesTotal") 
+
+    to_country_grand_total = tl_rec.get("VoteGrandTotal")
+    to_country_finishing_pos = tl_rec.get("FinishingPos")
+    
     # Country and Year are set as metadata elsewhere so don't need
     # to set them here -- however, do want DBpedia *Country* *URI*
@@ -115 +357 @@
     metadata_array.append({ "name": "Final",  "content": tl_final })
 
-    # tl_to_country_lc = tl_to_country.lower()
-    tl_to_country_uri = countrylabel_to_uri_mapping.get(tl_to_country)
+    ## tl_to_country_lc = tl_to_country.lower()
+    #tl_to_country_uri = countrylabel_to_uri_mapping.get(tl_to_country)
     if (tl_to_country_uri != None):
         metadata_array.append({ "name": "CountryDBURI","content": tl_to_country_uri })
-    else:
-        util.eprint("Warning: Failed to find DBpedia Country URI match for: " + tl_to_country)
-                
+#    else:
+#        util.eprint("Warning: Failed to find DBpedia Country URI match for: " + tl_to_country)
+
     for to_country_year_vote in to_country_year_votes:
         to_country   = to_country_year_vote.get('To country')
@@ -135 +377 @@
         metadata_array.append(voting_rec)
 
-        if (vote_type == "J"):
-            jury_metadata_vals.append(id_from_country+"-J")
-            jury_from_countries[id_from_country] = 1
-            to_country_jury_total = to_country_jury_total + points
-            
-        elif (vote_type == "T"):
-            tele_metadata_vals.append(id_from_country+"-T")
-            tele_from_countries[id_from_country] = 1
-            to_country_tele_total = to_country_tele_total + points
-        else:            
-            util.eprint("Warning: Unrecognized voting type: " + vote_type)
-
-    if (len(jury_metadata_vals)>0):
+#        if (vote_type == "J"):
+#            jury_metadata_vals.append(id_from_country+"-J")
+#            jury_from_countries[id_from_country] = 1
+#            to_country_jury_total = to_country_jury_total + points
+#            
+#        elif (vote_type == "T"):
+#            tele_metadata_vals.append(id_from_country+"-T")
+#            tele_from_countries[id_from_country] = 1
+#            to_country_tele_total = to_country_tele_total + points
+#        else:            
+#            util.eprint("Warning: Unrecognized voting type: " + vote_type)
+
+    metadata_array.append({ "name": "VoteMode", "content": vote_mode })
+ 
+    if ((jury_metadata_vals != None) and len(jury_metadata_vals)>0):
         metadata_array.append({ "name": "JuryVotesJSON", "content": json.dumps(jury_metadata_vals) })
         metadata_array.append({ "name": "JuryVotesTotal", "content": to_country_jury_total})
-    if (len(tele_metadata_vals)>0):
+    if ((tele_metadata_vals != None) and len(tele_metadata_vals)>0):
         metadata_array.append({ "name": "TeleVotesJSON", "content": json.dumps(tele_metadata_vals) })
         metadata_array.append({ "name": "TeleVotesTotal", "content": to_country_tele_total})
-                          
+    if ((comb_metadata_vals != None) and len(comb_metadata_vals)>0):
+        metadata_array.append({ "name": "CombVotesJSON", "content": json.dumps(comb_metadata_vals) })
+        metadata_array.append({ "name": "CombVotesTotal", "content": to_country_comb_total})
+    if ((sing_metadata_vals != None) and len(sing_metadata_vals)>0):
+        metadata_array.append({ "name": "SingVotesJSON", "content": json.dumps(sing_metadata_vals) })
+        metadata_array.append({ "name": "SingVotesTotal", "content": to_country_sing_total})
+
+    metadata_array.append({ "name": "VoteGrandTotal", "content": to_country_grand_total})
+    metadata_array.append({ "name": "FinishingPos", "content": to_country_finishing_pos})
+    
     filename_id = tl_to_country_id + "\\.nul"
 
@@ -193 +446 @@
         
     for to_country_year_votes in to_country_year_voting_groups:
-        
-        fileset = fileset_voting_for_esc_country_in_year(to_country_year_votes,countrylabel_to_uri_mapping)
+
+        augment_voting_for_esc_country_in_year(to_country_year_votes,countrylabel_to_uri_mapping)
+        fileset = fileset_voting_for_esc_country_in_year(to_country_year_votes)
         directory_metadata.append(fileset)
 
@@ -201 +455 @@
         
         print("  " + filename_id.ljust(28) + ": " + str(num_countries_voting_data) + " votes")
+
+
+    greenstone_metadata_json = { "DirectoryMetadata": directory_metadata }
+
+    return greenstone_metadata_json
+
+
+
+def augment_voting_in_year_then_country(to_country_year_votes,countrylabel_to_uri_mapping):
+
+    for to_year_votes in to_year_then_country_voting_groups:
+
+        for to_country_year_votes in to_year_votes:
+
+            augment_voting_for_esc_country_in_year(to_country_year_votes,countrylabel_to_uri_mapping)
+
+
+        to_year_votes.sort(key=lambda v: v[0].get("VoteGrandTotal"), reverse=True)
+        to_year_num_votes = len(to_year_votes)
+        for i in range(to_year_num_votes):
+            finishing_pos = i+1
+            to_year_votes[i][0]["FinishingPos"] = finishing_pos
+
+            
+def gs_yc_directory_metadata(to_year_then_country_voting_groups):
+    # Next step is to express the grouped to-country voting data
+    # in the Greenstone JSON metadata format:
+    
+    # { "DirectoryMetadata":
+    #   [
+    #     { "FileSet":
+    #       [
+    #         { "FileName": "France1991\.nul" },
+    #         { "Description":
+    #           {
+    #             "Metadata":
+    #              [
+    #                { "name": "Germany-J", "content": "12" }, # J = Jury Vote
+    #                  ...
+    #              ]
+    #           }
+    #         }
+    #       ]
+    #     }
+    #     ...
+    #    ]
+    #  }
+
+    directory_metadata = []
+
+    for to_year_votes in to_year_then_country_voting_groups:
+       
+        for to_country_year_votes in to_year_votes:
+
+##            augment_voting_for_esc_country_in_year(to_country_year_votes,countrylabel_to_uri_mapping)
+            fileset = fileset_voting_for_esc_country_in_year(to_country_year_votes)
+            directory_metadata.append(fileset)
+
+            filename_id = fileset.get('FileSet')[0].get('FileName')
+            num_countries_voting_data = len(fileset.get('FileSet')[1].get('Description').get('Metadata'))
+            
+            print("  " + filename_id.ljust(28) + ": " + str(num_countries_voting_data) + " votes")
 
 
@@ -223 +539 @@
         print("    <gsf:metadata name=\""+from_country+"-T\" />")
 
+    print()
+    for from_country in sorted(comb_from_countries.keys()):
+        print("    <gsf:metadata name=\""+from_country+"-C\" />")
+
+    print()
+    for from_country in sorted(sing_from_countries.keys()):
+        print("    <gsf:metadata name=\""+from_country+"-S\" />")
+        
     print("  </gsf:headMetaTags>")
 
@@ -271 +595 @@
 
 
-    to_country_year_voting_groups = create_to_country_voting_groups(data_hashmap_array_filtered)
+    # to_country_year_voting_groups = create_to_country_voting_groups(data_hashmap_array_filtered)
+    to_year_then_country_voting_groups = create_to_year_then_country_voting_groups(data_hashmap_array_filtered)
 
     # Debug output
     #
-    # print(to_country_year_voting_groups)
+    # print(to_year_then_country_voting_groups)
     
     print()
     print("Generating Greenstone JSON to-country voting metadata received by:")
-    greenstone_metadata_json = gs_directory_metadata(to_country_year_voting_groups,countrylabel_to_uri_mapping)
+    ##greenstone_metadata_json = gs_directory_metadata(to_country_year_voting_groups,countrylabel_to_uri_mapping)
+    augment_voting_in_year_then_country(to_year_then_country_voting_groups,countrylabel_to_uri_mapping)
+    greenstone_metadata_json = gs_yc_directory_metadata(to_year_then_country_voting_groups)
     
     print("Saving output as: " + json_output_filename)