Changeset 33202

Timestamp:

2019-06-22T17:13:38+12:00 (5 years ago)

Author:

davidb

Message:

Bound and Unbound CSV files changed to pring out all non-empty header columns

Location:

main/trunk/model-sites-dev/heritage-nz/collect/pdf-reports

Files:

• PREPARE-IMPORT.sh (modified) (1 diff)
• prepare/xls-to-csv.py (modified) (5 diffs)
• prepare/xls-to-sanitized-csv.py (modified) (1 diff)
• prepare/xlsutil.py (modified) (2 diffs)

main/trunk/model-sites-dev/heritage-nz/collect/pdf-reports/PREPARE-IMPORT.sh

@@ -7 +7 @@
 python prepare/xls-to-sanitized-csv.py \
   "import/Archaeological report PDFs 20190620.xlsx" \
-  "for-download/Archaeological-Reports-20190620.xlsx"
+  "for-download/Archaeological-Reports-20190620.csv"
 
 echo "Generating PDF-Bound and Unbound CSV files for use in import.pl"

main/trunk/model-sites-dev/heritage-nz/collect/pdf-reports/prepare/xls-to-csv.py

@@ -103 +103 @@
     # 3. Process the rest of the file, checking if accompanying 
     #     PDF file present or not and only printing out the header-names
-    #     specified in 'sanitized_headers'
+    #     as long as it has a non-empty header-name
+##    #     specified in 'sanitized_headers'
 
     num_rows = worksheet.nrows
@@ -117 +118 @@
 
         entry_utf8_row = []
-        row_i = 0;
-
-        for header_name in xlsutil.sanitized_headers:
-
-            if header_name in header_names_mapping:
+        num_header_cols = worksheet.row_len(xlsutil.header_row_pos)
+
+        for col_j in range(num_header_cols):
+##        for header_name in xlsutil.sanitized_headers:
+
+            header_name = worksheet.cell_value(xlsutil.header_row_pos,col_j)
+
+            if header_name and header_name.strip():
+####                print "*** 2. header_name = " + header_name
+
                 header_col_j = header_names_mapping[header_name]
-
-                header_cell_value = worksheet.cell_value(row_i,header_col_j)
+##            if header_name in header_names_mapping:
+##                header_col_j = header_names_mapping[header_name]
+
+##                header_cell_value = worksheet.cell_value(row_i,header_col_j)
                 
-                if header_cell_value == "Report Identifier":
-                    header_cell_value = "hnz.Identifier"
-
-                if header_cell_value == "Title":
-                    header_cell_value = "dc.Title"
-
-                if header_cell_value == "RelevantTLA's":
-                    header_cell_value = "TLA"
-
-                header_cell_value_utf8 = unicode(header_cell_value).encode("utf-8")
-                metadata_name_utf8 = header_cell_value_utf8.replace(" ", "")
+                if header_name == "Report Identifier":
+                    header_name = "hnz.Identifier"
+
+                if header_name == "Title":
+                    header_name = "dc.Title"
+
+                if header_name == "RelevantTLA's":
+                    header_name = "TLA"
+
+                header_name_utf8 = unicode(header_name).encode("utf-8")
+                metadata_name_utf8 = header_name_utf8.replace(" ", "")
 
                 entry_utf8_row.append(metadata_name_utf8)
 
-            else:
-                print("Warning: Failed to column mapping in spreadsheet for header name \""+header_name+"\" => skipping")
+##            else:
+##                print("Warning: Failed to column mapping in spreadsheet for header name \""+header_name+"\" => skipping")
 
         unbound_wr.writerow(entry_utf8_row)
@@ -151 +159 @@
         # 3. Process the rest of the file (metadata values) ...
 
-        row_i = row_i + 1
+        row_i = xlsutil.header_row_pos + 1
         while row_i<num_rows:
             num_cols = worksheet.row_len(row_i)
@@ -158 +166 @@
             found_pdf = False
 
-#            for col_j in range(num_cols):
-            for header_name in xlsutil.sanitized_headers:
-
-                if header_name in header_names_mapping:
+            for col_j in range(num_cols):
+##            for header_name in xlsutil.sanitized_headers:
+                header_name = worksheet.cell_value(xlsutil.header_row_pos,col_j)
+
+                if header_name and header_name.strip():
                     col_j = header_names_mapping[header_name]
+##                if header_name in header_names_mapping:
+##                    col_j = header_names_mapping[header_name]
 
                     cell_value = worksheet.cell_value(row_i,col_j)
@@ -201 +212 @@
 
                     formatted_utf8_row.append(formatted_cell_value_utf8)
-                else:
-                    print "Warning: No column number mapping for header name \""+header_name+"\" => skipping"
+##                else:
+##                    print "Warning: No column number mapping for header name \""+header_name+"\" => skipping"
 
             if found_pdf:

main/trunk/model-sites-dev/heritage-nz/collect/pdf-reports/prepare/xls-to-sanitized-csv.py

@@ -36 +36 @@
         exit(1)
 
-    # Move on to the start of the data values
-    row_i = 1
-
     # 2. Write out CSV file ...
+    # Work through all the row to the spreadsheet, including the header-names
+    row_i = xlsutil.header_row_pos 
+    
     while row_i<num_rows:
         # Work through sanitized header names building up row of utf8 values

main/trunk/model-sites-dev/heritage-nz/collect/pdf-reports/prepare/xlsutil.py

@@ -23 +23 @@
 
 
+header_row_pos = 0
+
 def getHeaderMappingToColNum(worksheet):
 
@@ -30 +32 @@
     header_names_mapping = {}
 
-    row_i = 0
-    num_header_cols = worksheet.row_len(row_i)
+    num_header_cols = worksheet.row_len(header_row_pos)
 
     found_header = False
     for col_j in range(num_header_cols):
-        header_cell_value = worksheet.cell_value(row_i,col_j)
+        header_cell_value = worksheet.cell_value(header_row_pos,col_j)
         if header_cell_value == "Report Identifier":
             found_header = True
 
-        header_names_mapping[header_cell_value] = col_j
+        if header_cell_value and header_cell_value.strip():
+            header_names_mapping[header_cell_value] = col_j
 
     if not found_header: