Changeset 38313 for gs3-installations/thewillow

Timestamp:

2023-10-15T23:58:43+13:00 (8 months ago)

Author:

davidb

Message:

Changes in response to running with the latest version of the Google drive sheet

Location:

gs3-installations/thewillow/trunk/sites/thewillow/collect/community-contributions/prepare

Files:

• 02-XLSX-TO-GSFRIENDLY-CSV.sh (modified) (1 diff)
• 06-UNIQUE-OBJECT-TYPES.sh (added)
• xlsx-to-csv--thewillow-directorysheet.py (modified) (4 diffs)

gs3-installations/thewillow/trunk/sites/thewillow/collect/community-contributions/prepare/02-XLSX-TO-GSFRIENDLY-CSV.sh

@@ -9 +9 @@
 echo "****"
 echo "!!!! Press ^C to abort !!!!"
+echo "Delete?: [Y/n]"
 echo "****"
-read -p "" -t 5
+read -p "" -t 5 do_delete
 
 echo "[... finished sleep]"
 
 echo ""
-echo "**** Removing: downloads/*"
-/bin/rm -rf downloads
-mkdir downloads
+
+if [ "x$do_delete" = "x" ] || [ "$do_delete" = "y" ] || [ "$do_delete" = "Y" ] ; then
+    echo "**** Removing: downloads/*"
+    /bin/rm -rf downloads
+    mkdir downloads
+else
+    echo "Prompt response: No"
+    echo "Skipping delete of: downloads/*"    
+fi
 
 echo ""

gs3-installations/thewillow/trunk/sites/thewillow/collect/community-contributions/prepare/xlsx-to-csv--thewillow-directorysheet.py

@@ -61 +61 @@
     
 
+ObjectTypeTransformation = {
+    "Audio Recording"               : "Audio",
+    "Interview Transcription"       : "Interview",
+    "Newspaper Article"             : "Newspaper",
+    "Newspaper Article (Microfilm)" : "Newspaper",
+    "Online Article"                : "Article",
+    "Photograph"                    : "Photograph",
+    "Written Recollection"          : "Memories"
+}
+    
+    
 
 def trimLastRow(worksheet):
@@ -161 +172 @@
             worksheet.cell(row=1,column=colpos, value=gs_heading)
 
+def transformObjectTypeValues(worksheet, num_rows,num_cols):
+    
+    # Skip header
+    row_i = 2
+
+    object_type_col_positions = [
+        "Object Type 1",
+        "Object Type 2",
+        "Object Type 3"
+    ]
+            
+    while row_i<=num_rows:
+
+        for object_type_heading in object_type_col_positions:
+            
+            object_type_colpos = HeadingsTransformation[object_type_heading]["colpos"]
+            
+            cell_val = worksheet.cell(row=row_i,column=object_type_colpos).value
+            
+            if cell_val is not None:
+                gs_cell_val = ObjectTypeTransformation[cell_val.strip()]
+                if gs_cell_val is not None:
+                    worksheet.cell(row=row_i,column=object_type_colpos, value=gs_cell_val)
+
+        row_i += 1                
 
 
@@ -185 +221 @@
     ofilename = os.path.join(downloads_outputdir,ofile)
 
-    # Do the actual downloading, and saving to file
-    response = requests.get(url)
-    open(ofilename, "wb").write(response.content)
-
-#    # The follow works for HTML or binary data
-#    with urlopen(url) as fin:
-#        data = fin.read()
-
-#    with open(ofilename, 'wb') as fout:
-#        fout.write(data)
+    if os.path.exists(ofilename):
+        print("  Skipping as file already downloaded")
+    else:    
+        # Do the actual downloading, and saving to file
+        response = requests.get(url)
+        open(ofilename, "wb").write(response.content)
+
+#        # The follow works for HTML or binary data
+#        with urlopen(url) as fin:
+#            data = fin.read()
+
+#        with open(ofilename, 'wb') as fout:
+#            fout.write(data)
     
     
@@ -458 +497 @@
     transformHeadings(worksheet,num_cols)
 
+    print("  Transforming Excel spreadsheet 'Object Type' values to Greenstone friendly ones (for indexing)")
+    transformObjectTypeValues(worksheet,num_rows,num_cols)
+
     print("  Processing values")