Changeset 33377 for gs3-extensions

Timestamp:

2019-07-31T19:04:00+12:00 (5 years ago)

Author:

ak19

Message:

Changes to get gen_SentenceDetection_model.sh to run still from the toplevel directory of this extenstion but located in bin/script.

Location:

gs3-extensions/maori-lang-detection

Files:

• README.txt (modified) (1 diff)
• gen_SentenceDetection_model.sh (modified) (4 diffs)

gs3-extensions/maori-lang-detection/README.txt

@@ -262 +262 @@
 https://stackoverflow.com/questions/36516363/sentence-detection-with-opennlp
 
+

gs3-extensions/maori-lang-detection/gen_SentenceDetection_model.sh

@@ -12 +12 @@
 # because the 2011 one appears to have fewer accidentally incorporated English sentences
 
+
+# Need to run this script from the top level folder of this extension
 
 if [ ! -z $1 ]; then
@@ -68 +70 @@
 #tail -100 $infile
 
+# Ensure OPENNLP_HOME is set
+if [ "x$OPENNLP_HOME" = "x" ]; then
+    echo "OPENNLP_HOME not set, attempting to set it to the local apache-opennlp (v1.9.1). ENSURE THIS EXISTS OR SET OPENNLP_HOME YOURSELF!"
+    #if [ -d apache-opennlp-* ]; then
+    cd apache-opennlp-*
+    if [ "x$?" = "x0" ]; then
+    export OPENNLP_HOME=`pwd`
+    cd ..
+    else
+    echo "No OPENNLP_HOME set and could not find a subfolder 'apache-opennlp-...' to set it to."
+    echo "Set OPENNLP_HOME yourself before running this script. Exitting..."
+    exit
+    fi
+fi
 
 mkdir -p $OPENNLP_HOME/training_data
@@ -100 +116 @@
 # Note that I tried manually inserting \t, after copying the original line with tabspacing had no effect. Still no difference.
 # Note 2: echo doesn't appear to preserve copied tab spaces.
-
+# Answer: echo doesn't treat \n as newline and \t as tab and so on, unless the -e flag is passed in:
+# echo -e "100000\tYWCA Boarding house : ĀwhinaServices and support Kei te pūmanawa o Tāmaki Makaurau a YMCA." | awk -F "\t" '{ print $2 }'
 
 # 2. Create mri sentences model from training sentences file
 #$OPENNLP_HOME/bin/opennlp SentenceDetectorTrainer -model mri-sent_trained.bin -lang en -data mri-sent.train -encoding UTF-8
 
-if [ "x$OPENNLP_HOME" = "x" ]; then
-    echo "OPENNLP_HOME not set, attempting to set it to apache-opennlp-1.9.1 (ENSURE THIS EXISTS OR SET OPENNLP_HOME YOURSELF!)"
-    if [ -d apache-opennlp-* ]; then
-    cd apache-opennlp-*
-    export OPENNLP_HOME=`pwd`
-    cd ..
-    else
-    echo "No OPENNLP_HOME set and could not find a subfolder 'apache-opennlp-...' to set it to."
-    echo "Set OPENNLP_HOME yourself before running this script. Exitting..."
-    fi
-fi
 
 mkdir -p $OPENNLP_HOME/models
@@ -132 +138 @@
 echo "****************************"
 echo ""
+