Changeset 35070

Timestamp:

2021-04-14T17:24:52+12:00 (3 years ago)

Author:

davidb

Message:

Introduction of JT for combined Jury and Tele vote metedata fromcountry entries (for years 2016-) where Jury and Tele are separate

Location:

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare

Files:

• 03-GEN-VOTING-METADATA.sh (modified) (1 diff)
• voting-excel/xlsx-fromcountry-jsonmetadata.py (modified) (6 diffs)
• voting-excel/xlsxutil.py (modified) (2 diffs)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/03-GEN-VOTING-METADATA.sh

@@ -61 +61 @@
 
     if [ $? = 0 ] ; then
-    $prep_dir/xlsx-tocountry-jsonmetadata.py $prep_dir/eurovision_song_contest_1975_2019.xlsx  $prep_dir/metadata-votes-tocountry.json
 
-    if [ $? != 0 ] ; then
+    $prep_dir/xlsx-fromcountry-jsonmetadata.py \
+        --votingtype "JT" \
+        $prep_dir/eurovision_song_contest_1975_2019.xlsx  \
+        $prep_dir/metadata-votes/metadata-votes-fromcountry-comb.json
+
+    if [ $? = 0 ] ; then    
+        $prep_dir/xlsx-tocountry-jsonmetadata.py $prep_dir/eurovision_song_contest_1975_2019.xlsx  $prep_dir/metadata-votes-tocountry.json
+
+        if [ $? != 0 ] ; then
+        echo "!!!!" 1>&2
+        echo "! Error enountered generating to-country metadata-json: $prep_dir/metadata-votes-tocountry.json" 1>&2
+        echo "!!!!" 1>&2
+        exit_status=1
+        fi
+    else
         echo "!!!!" 1>&2
-        echo "! Error enountered generating to-country metadata-json: $prep_dir/metadata-votes-tocountry.json" 1>&2
+        echo "! Error enountered generating from-country combined Jury and Televoting metadata-json in $prep_dir/metadata-votes/" 1>&2
         echo "!!!!" 1>&2
         exit_status=1

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/voting-excel/xlsx-fromcountry-jsonmetadata.py

@@ -17 +17 @@
 #tele_from_countries = {}
     
+def list_jury_tele_comb_votes(data_hashmap_array):
+
+    data_hashmap_array_filtered = list(filter(xlsxutil.filter_jury_or_tele_votes, data_hashmap_array))
+    data_hashmap_array_filtered.sort(key=xlsxutil.sortkey_year_from_country_to_country_voting_type)
+    data_hashmap_array_filtered_len = len(data_hashmap_array_filtered)
+    
+    data_hashmap_array_filtered_comb_pairs = []
+
+    i = 0;
+    while (i<data_hashmap_array_filtered_len-1):
+        entry = data_hashmap_array_filtered[i];
+        next_entry = data_hashmap_array_filtered[i+1];
+
+        if ((entry.get("Year") == next_entry.get("Year")) and (entry.get("From country") == next_entry.get("From country"))  
+            and ((entry.get("Jury or Televoting") == "J") and (next_entry.get("Jury or Televoting") == "T"))):
+
+            comb_entry = entry.copy()
+            comb_entry["Points"] = entry.get("Points") + next_entry.get("Points")
+            comb_entry["Jury or Televoting"] = "JT"
+
+            from_country = entry.get("From country")
+            year = entry.get("Year")
+            from_country_id = "FromCountry-" + re.sub(r'\s+', '', from_country) + str(year)
+                
+            comb_entry["FromCountry-Child-Jury"] = from_country_id + "-J"
+            comb_entry["FromCountry-Child-Tele"] = from_country_id + "-T"
+            
+            data_hashmap_array_filtered_comb_pairs.append(comb_entry)
+            i = i + 2 
+        else:
+            i = i + 1
+            
+    return data_hashmap_array_filtered_comb_pairs
+
 
 def create_from_country_voting_groups(data_hashmap_array):
@@ -31 +65 @@
     # Sort so array entries are grouped by the country receiving the votes in a given year
     data_hashmap_array.sort(key=xlsxutil.sortkey_year_from_country)
+
+
+    # Debug output
+    #
+    # for data_hashmap in data_hashmap_array:
+    #    print(data_hashmap.get('From country'), data_hashmap.get('Year'), data_hashmap.get('Points'), "(Points to " + data_hashmap.get('To country') + ")")
+
+
+    # Build array of country groups
+    #   A country grouping includes all the votes that country receive that year
+    country_groups = []
+
+    prev_data_hashmap = data_hashmap_array[0]
+    country_group = [ ]
+
+    i = 1
+    num_rows = len(data_hashmap_array)
+
+    while (i < num_rows):        
+        country_group.append(prev_data_hashmap)
+        data_hashmap = data_hashmap_array[i]
+
+        if (data_hashmap.get('From country') != prev_data_hashmap.get('From country')):
+            # moving on to a new country group
+            country_groups.append(country_group)
+            country_group = [ ]
+            
+        prev_data_hashmap = data_hashmap
+        i = i + 1
+        
+    country_group.append(prev_data_hashmap)        
+    country_groups.append(country_group)
+
+    return country_groups
+
+
+
+
+def XXXXcreate_from_country_voting_pairs_parent(data_hashmap_array):
+
+    # Example values for header-names
+    #   (semi-) final: f, sf
+    #   Jury or Televoting: J, T
+    #   Year: 1975
+    #   To country: Belgium
+    #   From country: Belgium
+    #   Edition: 1975f, 1975sf
+
+    
+    # Sort so array entries are grouped by the country receiving the votes in a given year
+    data_hashmap_array.sort(key=xlsxutil.sortkey_year_from_country_voting_type)
 
 
@@ -132 +217 @@
     metadata_array.append({"name": "FromCountryVoteMode",    "content" : vote_mode})
     metadata_array.append({"name": "FromCountryNuancedType", "content" : nuanced_vote_type})
+
+    if tl_rec.get("FromCountry-Child-Jury") is not None:
+        metadata_array.append({"name": "FromCountryChildJury", "content" : tl_rec["FromCountry-Child-Jury"]})
+    if tl_rec.get("FromCountry-Child-Tele") is not None:
+        metadata_array.append({"name": "FromCountryChildTele", "content" : tl_rec["FromCountry-Child-Tele"]})
+
     
     for from_country_year_vote in from_country_year_votes:
@@ -225 +316 @@
     parser = argparse.ArgumentParser()
     parser.add_argument('--sheetname', help="The name of the sheet within the Excel file to extractc data from")
-    parser.add_argument('--votingtype', choices=['J','T'], help="Filter to only J=Jury or T=Tele cast votes")
+    parser.add_argument('--votingtype', choices=["J","T", "JT"], help="Filter to only J=Jury, T=Tele cast votes, JT=Combined jury and tele votes")
     parser.add_argument('input-file.xlsx')
     parser.add_argument('output-file.json', nargs='?')
@@ -254 +345 @@
     if voting_type != None:
         # Further filter down by the type of voting results cast
-        if (voting_type == "J"):
+        if (voting_type == "JT"):
+            data_hashmap_array_filtered = list_jury_tele_comb_votes(data_hashmap_array_filtered)
+            print("Number Jury-Tele *combined* cast data rows: " + str(len(data_hashmap_array_filtered)))
+        elif (voting_type == "J"):
             data_hashmap_array_filtered = list(filter(xlsxutil.filter_jury_votes, data_hashmap_array_filtered))
             print("Number Jury cast data rows:          " + str(len(data_hashmap_array_filtered)))
@@ -264 +358 @@
     print()
 
+    from_country_year_voting_groups = {}
+    
+#    if ((voting_type != None) and (voting_type == "JT")):
+#        from_country_year_voting_groups = create_from_country_combined_voting_groups(data_hashmap_array_filtered)
+#    else:
+#        from_country_year_voting_groups = create_from_country_voting_groups(data_hashmap_array_filtered)
+
     from_country_year_voting_groups = create_from_country_voting_groups(data_hashmap_array_filtered)
+        
     debug_print_country_groups(from_country_year_voting_groups)
-
-    # Debug output
-    #
-    # print(from_country_year_voting_groups)
 
     print()

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/voting-excel/xlsxutil.py

@@ -45 +45 @@
     return entry.get("Jury or Televoting") == "T" and entry.get('Duplicate') == None
 
+def filter_jury_or_tele_votes(entry):
+    return (entry.get("Jury or Televoting") == "J" or entry.get("Jury or Televoting") == "T") and entry.get('Duplicate') == None
+
 
 def sortkey_year_to_country(entry):
@@ -51 +54 @@
 def sortkey_year_from_country(entry):
     return str(entry.get('Year'))+entry.get('From country')
+
+def sortkey_year_from_country_to_country_voting_type(entry):
+    return str(entry.get('Year'))+entry.get('From country')+entry.get('To country')+entry.get('Jury or Televoting')
+