Changeset 33498

Timestamp:

2019-09-23T16:43:22+12:00 (5 years ago)

Author:

ak19

Message:

Corrections to script. Modified the tests checking for file/dir existence on hdfs. The use of direct hdfs commands return values worked on the command line but not in the script. So have used another way suggested on stackoverflow that does appear to work in the intended manner.

File:

• gs3-extensions/maori-lang-detection/bin/script/get_maori_WET_records_for_crawl.sh (modified) (3 diffs)

gs3-extensions/maori-lang-detection/bin/script/get_maori_WET_records_for_crawl.sh

@@ -74 +74 @@
 
 # https://stackoverflow.com/questions/26513861/checking-if-directory-in-hdfs-already-exists-or-not
-hdfs dfs -test -d $OUTPUT_PARENTDIR
-if [ $? == 0 ]; then
+#hdfs dfs -test -d $OUTPUT_PARENTDIR
+#if [ $? == 0 ]; then
+if $(hdfs dfs -test -d "$OUTPUT_PARENTDIR"); then
     echo "Directory $OUTPUT_PARENTDIR already exists."        
 else
@@ -131 +132 @@
 # The above generates ZIPPED part*.csv files in $OUTPUTDIR (folder cc-mri-csv).
 # First create a folder and unzip into it:
-hdfs dfs -test -d $OUTPUT_PARENTDIR/cc-mri-unzipped-csv
-if [ $? == 0 ]; then
+#hdfs dfs -test -d $OUTPUT_PARENTDIR/cc-mri-unzipped-csv
+#if [ $? == 0 ]; then
+if $(hdfs dfs -test -d "$OUTPUT_PARENTDIR/cc-mri-unzipped-csv"); then
     echo "Directory cc-mri-unzipped-csv already exists for crawl ${CRAWL_ID}."
     echo "Assuming cc-mri.csv also exists inside $OUTPUT_PARENTDIR"
@@ -172 +174 @@
 # PHASE 3: convert warc files to wet files and copy the wet files into the mounted shared area
 
-hdfs dfs -test -f $OUTPUTDIR/_SUCCESS
-if [ $? == 0 ]; then
+#hdfs dfs -test -f $OUTPUTDIR/_SUCCESS
+#if [ $? == 0 ]; then
+if $(hdfs dfs -test -f "$OUTPUTDIR/_SUCCESS"); then
     # ia-hadoop-tools converts warc files into wet (and wat) files but expects a particular folder structure
     # Create the expected folder structure: a "wet" and a "wat" folder should exist