Changeset 37649

Timestamp:

2023-04-11T23:52:30+12:00 (13 months ago)

Author:

davidb

Message:

Building out of default/model collection to better suit The Willow imported data

Location:

gs3-installations/thewillow/trunk/sites/thewillow/collect/community-contributions

Files:

• RECONFIGURE.sh (modified) (1 diff)
• etc/collectionConfig.xml (modified) (5 diffs)
• prepare/README.txt (modified) (1 diff)
• prepare/xlsx-to-csv--thewillow-directorysheet.py (modified) (5 diffs)

gs3-installations/thewillow/trunk/sites/thewillow/collect/community-contributions/RECONFIGURE.sh

@@ -1 +1 @@
 #!/bin/bash
 
+source ../_local_collect_config.bash
+
 wget -O - "${gs_library_url}?a=s&sa=c&sc=$col" >/dev/null

gs3-installations/thewillow/trunk/sites/thewillow/collect/community-contributions/etc/collectionConfig.xml

@@ -32 +32 @@
       </xsl:template>   
     </format>
-    <search type="lucene">
+    <search type="solr">
         <level name="document">
             <displayItem lang="en" name="name">document</displayItem>
         </level>
+        <!--
         <level name="section">
             <displayItem lang="en" name="name">section</displayItem>
         </level>
+        -->
         <defaultLevel name="document"/>
-        <index name="text">
-            <displayItem lang="en" name="name">text</displayItem>
+        <index name="text,Title,Creator,Description,Notes">
+          <displayItem lang="en" name="name">all text</displayItem>
         </index>
         <index name="dc.Title,ex.dc.Title,Title">
-            <displayItem lang="en" name="name">titles</displayItem>
+          <displayItem lang="en" name="name">titles</displayItem>
         </index>
-        <index name="Source">
-            <displayItem lang="en" name="name">filenames</displayItem>
+        <index name="Creator">
+          <displayItem lang="en" name="name">content creators</displayItem>
         </index>
-        <defaultIndex name="text"/>
+        <index name="Description">
+          <displayItem lang="en" name="name">descriptions</displayItem>
+        </index>
+        <index name="Notes">
+          <displayItem lang="en" name="name">notes</displayItem>
+        </index>
+        <defaultIndex name="text,Title,Creator,Description,Notes"/>
+
         <sort name="rank">
           <displayItem lang="en" name="name">rank</displayItem>
                 </sort>
+        <!--
                 <sort name="none">
           <displayItem lang="en" name="name">natural (build) order</displayItem>
         </sort>
+        -->
+        
+        <facet name="InCopyright">
+          <displayItem lang="en" name="name">In Copyright</displayItem>       
+        </facet>
+        <facet name="Theme">
+          <displayItem lang="en" name="name">Theme</displayItem>          
+        </facet>
+        <facet name="ObjectType">
+          <displayItem lang="en" name="name">Content Type</displayItem>       
+        </facet>
+        
         <searchType name="plain"/>
         <searchType name="simpleform"/>
@@ -81 +103 @@
             <plugin name="ZIPPlugin"/>
             <plugin name="GreenstoneXMLPlugin"/>
+
             <plugin name="CSVPlugin"/>
+            <plugin name="ImagePlugin">
+              <option name="enable_cache"/>
+            </plugin>
             <plugin name="TextPlugin"/>
             <plugin name="HTMLPlugin"/>
-            <plugin name="EmailPlugin"/>
+            <!-- <plugin name="EmailPlugin"/>  -->
+            
             <plugin name="PDFv2Plugin"/>
             <!-- Configuring an UnknownConverterPlugin for docx processing with Tika -->
@@ -117 +144 @@
             <plugin name="PowerPointPlugin"/>
             <plugin name="ExcelPlugin"/>
-            <plugin name="ImagePlugin"/>
             <plugin name="ISISPlugin"/>
             <plugin name="NulPlugin"/>
@@ -126 +152 @@
         </pluginList>
     </import>
+    <importOption name="OIDtype"     value="assigned"/>
+    <importOption name="OIDmetadata" value="DLIdentifier"/>
+    
     <browse>
         <classifier name="List">
@@ -244 +273 @@
                 (e.g. hierachical and paged documents) as it can prevent any sub-sections from showing.
             -->
-            <!-- 
+
+            <gsf:template name="documentPre">
+              <!--
+              Title
+              Creator
+              Description
+              CreationDate
+              OirginalOrCopy
+              FileNote
+              LibraryRefNum
+              InCopyright
+              Theme Theme   Theme
+              ObjectType    ObjectType  ObjectType
+              Notes
+              OptHyperlink
+              -->
+              <br/>
+              <gsf:metadata-table>
+                <!-- <gsf:metadata name="Title">Title:</gsf:metadata> -->
+                <gsf:metadata name="Creator">Creator:</gsf:metadata>
+                <gsf:metadata name="Description">Description:</gsf:metadata>
+                <gsf:metadata name="OirginalOrCopy">Original or Copy:</gsf:metadata>
+                <gsf:metadata name="InCopyright">In Copyright:</gsf:metadata>
+                <gsf:metadata name="Theme">Theme(s):</gsf:metadata>
+                <gsf:metadata name="DLIdentifier">Document Id:</gsf:metadata>
+                <gsf:metadata name="Notes">Notes:</gsf:metadata>
+              </gsf:metadata-table>
+              <br/>
+              
+              <gsf:switch>
+                <gsf:metadata name="OptHyperlink"/>
+                <gsf:when test="exists">
+
+
+                  <gsf:switch>
+                <gsf:metadata name="Plugin"/>
+                <gsf:when test="equals" test-value="ImagePlugin">
+                  <!-- supress external link, as screenview with link to original displayed -->
+                </gsf:when>
+                <gsf:otherwise>
+                  <div>
+                    <a target="_blank">
+                      <xsl:attribute name="href"><gsf:metadata name="OptHyperlink"/></xsl:attribute>
+                      View original content
+                    </a>
+                  </div>
+                </gsf:otherwise>
+                  </gsf:switch>
+                  
+                </gsf:when>
+              </gsf:switch>
+
+            </gsf:template>
+
+            <!--
             <gsf:template name="documentContent">
               <div id="gs-document">
                 <xsl:call-template name="documentPre"/>
                 <xsl:call-template name="wrappedSectionImage"/>
-                <div id="gs-document-text">
-                <xsl:call-template name="documentNodeText"/>
-                </div>
+
+                <gsf:switch>
+                  <gsf:metadata name="FileFormat"/>
+                  <gsf:when test="equals" test-value="HTML">
+                  </gsf:when>
+                  <gsf:otherwise>
+                <div id="gs-document-text">
+                  <xsl:call-template name="documentNodeText"/>
+                </div>
+                  </gsf:otherwise>
+                </gsf:switch>
                   </div>            
             </gsf:template>

gs3-installations/thewillow/trunk/sites/thewillow/collect/community-contributions/prepare/README.txt

@@ +1 @@
+
+#----
+# To be able to convert docx to pdf
+#----
+
+# headless/minimal version
+
+sudo apt-get --no-install-recommends install libreoffice
+
+
+#----
+# To process Excel spreadsheet and download Google drive
+#----
 
 pip install openpyxl
 pip install gdown
+
+pip install requests
 
 

gs3-installations/thewillow/trunk/sites/thewillow/collect/community-contributions/prepare/xlsx-to-csv--thewillow-directorysheet.py

@@ -8 +8 @@
 import gdown
 import openpyxl
-#import urllib.request
-
+
+# import urllib.request
 # from urllib.request import urlopen
 
@@ -23 +23 @@
 # The '/' on the end is important for gdown 
 downloads_outputdir = "downloads/"
-
-#import ssl
-#ssl._create_default_https_context = ssl._create_unverified_context
 
 
@@ -169 +166 @@
 
     ofile = url.rsplit('/', 1)[-1] # everything after the last '/'
+    if ofile == "":
+        # url ended in a diretory separator
+        # assume web server serves up default directory listing (e.g. index.html) page
+        ofile="index.html"
+        
     file_ext = os.path.splitext(ofile)[1]
-
+        
     # Looking for a simple file extension to the URL, otherwise assume HTML
 
@@ -210 +212 @@
     return download_filename
 
+def optConvertDocToPdf(src_filename):
+
+    # Default assumption, no conversion
+    #
+    dst_filename = src_filename
+    
+    src_dirname  = os.path.dirname(src_filename)    
+    src_filename_root, file_ext = os.path.splitext(src_filename)
+
+    lc_file_ext = file_ext.lower()
+    
+    if lc_file_ext == ".docx" or lc_file_ext == ".doc":
+
+        dst_filename = src_filename_root + ".pdf"
+        
+        cmd = "soffice --headless --convert-to pdf \"" + src_filename + "\" --outdir \"" + src_dirname + "\""
+        status = os.system(cmd)
+
+        if status == 0:
+            os.remove(src_filename)
+
+    return dst_filename
+
+    
 def fileRenameToDLIdentifier(downloaded_filename,dl_identifier_str):
     
@@ -260 +286 @@
             orig_filename_colpos = HeadingsTransformation["OrigFilename"]["colpos"]
             worksheet.cell(row=row_i, column=orig_filename_colpos, value=orig_file)
+
+            # If a doc/docx file, convert it to PDF with open-/ libre-office
+            # (assumed to be installed, otherwise conversion fails,
+            #  and we continue with doc/docx file)
+            #
+            downloaded_filename = optConvertDocToPdf(downloaded_filename)
 
             renamed_filename = fileRenameToDLIdentifier(downloaded_filename,dl_identifier_str)