Changeset 34861

Timestamp:

2021-02-15T16:37:20+13:00 (3 years ago)

Author:

davidb

Message:

General code moved to module

Location:

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/voting-excel

Files:

• xlsx-to-jsonmetadata.py (modified) (3 diffs)
• xlsxutil.py (added)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/voting-excel/xlsx-to-jsonmetadata.py

@@ -6 +6 @@
 import argparse
 
-# import xlrd
 import openpyxl
 
-#import csv
-
-#import xlsutil
-
-#filter_headers = [
-#    u"Heading 1", u"Heading 2"
-#    # ... and so on
-#]
-
-filter_headers = None
-
-def load_xslx_sheet(excel_input_filename,sheetname):
-    print "Loading: " + excel_input_filename
-    print "  ..."
-    workbook = openpyxl.load_workbook(excel_input_filename, data_only=True)
-    print "  ......Done"
-
-    print "  Loaded worksheets: ",  workbook.sheetnames
-
-    worksheet = None
-    
-    if (sheetname == None):
-        print "  Selecting default/active worksheet"
-        worksheet = workbook.active
-    else:
-        print "  Selecting worksheet '"+sheetname+"'"
-        worksheet = workbook[sheetname]
-
-    print ""
-    return worksheet
-
-
-def header_to_column_mapping(worksheet,header_row_pos):
-
-    # For header line, build up hashmap of header-names to column number
-
-    header_names_mapping = {}
-
-    min_col = worksheet.min_column
-    max_col = worksheet.max_column
-    
-    found_header = False
-    for col_j in range(min_col,max_col+1):
-        header_cell = worksheet.cell(row=header_row_pos, column=col_j)
-        header_cell_value = header_cell.value.strip()
-        
-        if header_cell_value:
-            header_names_mapping[header_cell_value] = col_j
-
-    return header_names_mapping
-
-
-def convert_worksheet_to_hashmaps(worksheet):
-
-    print "Converting worksheet to array header-name keyed array of hashmaps"
-    
-    data_hashmap_array = [];
-
-    # scan through rows until non-empty/headernames encountered
-
-    header_row_pos = None;
-    
-    row_i = 1
-    for row in worksheet.values:
-        for value in row:
-            if (value != None) :
-                header_row_pos = row_i
-                break
-        
-            print(value)
-        if (header_row_pos != None): break
-        row_i = row_i + 1
-     
-    print "  Found headername data in row: ", header_row_pos
-        
-    header_names_mapping = header_to_column_mapping(worksheet,header_row_pos)
-    header_names = header_names_mapping.keys()
-
-    # Move on to next row, to start processing the data values
-    row_i = row_i + 1
-    num_rows = worksheet.max_row;
-    
-    while row_i<=num_rows:
-        # Work through the provided header names, building up hashmap of header-name to value
-        data_hashmap = {}
-        
-        for header_name in header_names:
-
-            header_col_j = header_names_mapping[header_name]
-
-            data_cell = worksheet.cell(row=row_i,column=header_col_j)
-            data_value = data_cell.value
-
-            data_hashmap[header_name] = data_value
-
-        row_i = row_i + 1
-        
-        data_hashmap_array.append(data_hashmap)
-
-    print ""
-    return data_hashmap_array
+import xlsxutil
+
 
 def fileset_voting_for_esc_country_in_year(data_hashmap):
@@ -247 +147 @@
         json_output_filename = os.path.splitext(excel_input_filename)[0]+'.json'
         
-    worksheet = load_xslx_sheet(excel_input_filename,sheetname)
-
-    data_hashmap_array = convert_worksheet_to_hashmaps(worksheet)
+    worksheet = xlsxutil.load_xslx_sheet(excel_input_filename,sheetname)
+
+    data_hashmap_array = xlsxutil.convert_worksheet_to_hashmaps(worksheet)
 
     print "Number of data rows: ", len(data_hashmap_array)
 
+    
     to_country_year_voting_groups = create_to_country_voting_groups(data_hashmap_array)
 
@@ -286 +187 @@
     ### TODO
 
-    ## fix up names to match those used in DBpedia ESC
     ## provide metadata value that lists the countries-year-type to voted
 
-    # F.Y.R.Macedonia2012
-    # Serbia&Montenegro2004.id    : 35 votes
-    # Bosnia&Herzegovina2005.id   : 38 votes
-    # TheNetherlands 
-    
-
-    # From ESC:
-    #   Republic of Macedonia (2005)
-    #   Netherlands (not The Netherlands)
-    #   Federal Republic of Yugoslavia (1992)
-    
-    
     print "Creating Greenstone JSON voting metadata for:"
     for to_country_year_votes in to_country_year_voting_groups: