Changeset 33222 for main

Timestamp:

2019-06-26T15:52:27+12:00 (5 years ago)

Author:

davidb

Message:

More careful working with unicode/utf8

Location:

main/trunk/model-sites-dev/heritage-nz/collect/pdf-reports/prepare

Files:

• xls-to-csv.py (modified) (5 diffs)
• xls-to-sanitized-csv.py (modified) (1 diff)
• xlsutil.py (modified) (2 diffs)

main/trunk/model-sites-dev/heritage-nz/collect/pdf-reports/prepare/xls-to-csv.py

@@ -124 +124 @@
 ##        for header_name in xlsutil.sanitized_headers:
 
-            header_name = worksheet.cell_value(xlsutil.header_row_pos,col_j)
-
-            if header_name and header_name.strip():
+            header_name = xlsutil.cell_value_tidy_unicode(worksheet,xlsutil.header_row_pos,col_j)
+
+            if header_name:
 
                 header_col_j = header_names_mapping[header_name]
@@ -132 +132 @@
 ##                header_col_j = header_names_mapping[header_name]
 
-##                header_name = worksheet.cell_value(row_i,header_col_j)
+##                header_name = xslutil.cell_value_tidy_unicode(worksheet,row_i,header_col_j)
                 
-                if header_name == "Report Identifier":
-                    header_name = "hnz.Identifier"
-
-                if header_name == "Title":
-                    header_name = "dc.Title"
-
-                if header_name == "Relevant TLA's":
-                    header_name = "TLA"
-
-                header_name_utf8 = unicode(header_name).encode("utf-8")
+                if header_name == u"Report Identifier":
+                    header_name = u"hnz.Identifier"
+
+                if header_name == u"Title":
+                    header_name = u"dc.Title"
+
+                if header_name == u"Relevant TLA's":
+                    header_name = u"TLA"
+
+#                header_name_utf8 = unicode(header_name).encode("utf-8")
+                header_name_utf8 = header_name.encode("utf-8")
                 metadata_name_utf8 = header_name_utf8.replace(" ", "")
 
                 entry_utf8_row.append(metadata_name_utf8)
 
-                if header_name == "Site No":
-                    entry_utf8_row.append("SiteNoOrdering")
+                if header_name == u"Site No":
+                    entry_utf8_row.append(u"SiteNoOrdering")
 
 ##            else:
@@ -171 +172 @@
             for col_j in range(num_cols):
 ##            for header_name in xlsutil.sanitized_headers:
-                header_name = worksheet.cell_value(xlsutil.header_row_pos,col_j)
-
-                if header_name and header_name.strip():
+                header_name = xlsutil.cell_value_tidy_unicode(worksheet,xlsutil.header_row_pos,col_j)
+
+                if header_name:
                     col_j = header_names_mapping[header_name]
 ##                if header_name in header_names_mapping:
 ##                    col_j = header_names_mapping[header_name]
 
-                    cell_value = worksheet.cell_value(row_i,col_j)
-
-                    formatted_cell_value = xlsutil.format_if_int(cell_value)
-
-                    if header_name == "Report Identifier":
+                    formatted_cell_value = xlsutil.cell_value_tidy_unicode(worksheet,row_i,col_j)
+
+                    if header_name == u"Report Identifier":
                         # Check to see if companion PDF file present
                         # pdf_filename = os.path.join(excel_dirname,"pdfs",formatted_cell_value+".pdf")
@@ -208 +207 @@
                         formatted_cell_value = formatted_cell_value.rstrip();
 
-                    formatted_cell_value_utf8 = unicode(formatted_cell_value).encode("utf-8")
+#                    formatted_cell_value_utf8 = unicode(formatted_cell_value).encode("utf-8")
+                    formatted_cell_value_utf8 = formatted_cell_value.encode("utf-8")
 
                     ## Perform any cell transformations to make DL used spreadsheet
@@ -216 +216 @@
                     formatted_utf8_row.append(formatted_cell_value_utf8)
 
-                    if header_name == "Site No":
+                    if header_name == u"Site No":
                         site_no = formatted_cell_value_utf8
                         site_no_ordering = None

main/trunk/model-sites-dev/heritage-nz/collect/pdf-reports/prepare/xls-to-sanitized-csv.py

@@ -49 +49 @@
                 header_col_j = header_names_mapping[header_name]
 
-                cell_value = worksheet.cell_value(row_i,header_col_j)
+#                cell_value = worksheet.cell_value(row_i,header_col_j)
+#
+#                cell_value_utf8 = unicode(cell_value).encode("utf-8")
 
-                cell_value_utf8 = unicode(cell_value).encode("utf-8")
+                cell_value_unicode = xlsutil.cell_value_tidy_unicode(worksheet,row_i,header_col_j)
+
+                cell_value_utf8 = cell_value_unicode.encode("utf-8")
+
+##                cell_value_utf8 = unicode(cell_value).encode("utf-8")
 
                 sanitized_utf8_row.append(cell_value_utf8)

main/trunk/model-sites-dev/heritage-nz/collect/pdf-reports/prepare/xlsutil.py

@@ -6 +6 @@
 ## worksheet_name = "Archaeological reports"
 
-sanitized_headers = [ "Report Identifier", "Author", "Title", "Produced By", "Date", "HNZPT Region", "File No",
-                      "Authority No", "Site No", "Relevant TLA's", "Record Type", "Date Entered" ]
+sanitized_headers = [ u"Report Identifier", u"Author", u"Title", u"Produced By", u"Date", u"HNZPT Region", u"File No",
+                      u"Authority No", u"Site No", u"Relevant TLA's", u"Record Type", u"Date Entered" ]
+
+def make_unicode(value):
+    value_unicode = value
+
+    if type(value) != unicode:
+        value_str = value
+        if type(value_str) != str:
+            value_str = str(value)
+        
+        value_unicode =  value_str.decode('utf-8')
+
+    return value_unicode
+
+
+def cell_value_tidy_unicode(worksheet,row_i,col_j):
+    value= worksheet.cell_value(row_i,col_j)
+
+    value_unicode = make_unicode(value)
+    value_unicode_stripped = value_unicode.strip()
+
+    return value_unicode_stripped
+
 
 def format_if_int(cell_value):
@@ -36 +58 @@
     found_header = False
     for col_j in range(num_header_cols):
-        header_cell_value = worksheet.cell_value(header_row_pos,col_j)
-        if header_cell_value == "Report Identifier":
+        header_cell_value = cell_value_tidy_unicode(worksheet,header_row_pos,col_j)
+        if header_cell_value == u"Report Identifier":
             found_header = True