Changeset 35961

Timestamp:

2022-01-11T14:49:25+13:00 (2 years ago)

Author:

davidb

Message:

Improvements to the running of the script, and its underlying Python component

Location:

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare

Files:

• 07b-GENERATE-ESSENTIA-FEATURES-DATA--ALL-BATCHx8--EACH-A-NOHUP.sh (modified) (1 diff)
• 07b-GENERATE-ESSENTIA-FEATURES-DATA--SMALL.sh (modified) (1 diff)
• 07b-GENERATE-ESSENTIA-FEATURES-DATA.sh (modified) (2 diffs)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/07b-GENERATE-ESSENTIA-FEATURES-DATA--ALL-BATCHx8--EACH-A-NOHUP.sh

@@ -11 +11 @@
 
 nohup ./07b-GENERATE-ESSENTIA-FEATURES-DATA.sh "compute_audio_features" --start 2005 --end 2012 > nohup-compute-features-block7.out 2>&1 &
-nohup ./07b-GENERATE-ESSENTIA-FEATURES-DATA.sh "compute_audio_features" --start 2013 --end 2021 > nohup-compute-features-block8.out 2>&1 &
+nohup ./07b-GENERATE-ESSENTIA-FEATURES-DATA.sh "compute_audio_features" --start 2013 --end $esc_cutoff_endyear > nohup-compute-features-block8.out 2>&1 &

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/07b-GENERATE-ESSENTIA-FEATURES-DATA--SMALL.sh

@@ -11 +11 @@
 
 ./07b-GENERATE-ESSENTIA-FEATURES-DATA.sh "scrape_contestants" --start $year --end $year
-# ./07b-GENERATE-ESSENTIA-FEATURES-DATA.sh "download_audio|compute_audio_features" --start $year --end $year contestants-$year.csv
+./07b-GENERATE-ESSENTIA-FEATURES-DATA.sh "download_audio" --start $year --end $year contestants-$year.csv
+./07b-GENERATE-ESSENTIA-FEATURES-DATA.sh "compute_audio_features" --start $year --end $year
+
 
     

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/prepare/07b-GENERATE-ESSENTIA-FEATURES-DATA.sh

@@ -97 +97 @@
 export PATH=$PWD/$prep_dir:$PATH
        
+run_scrape_status=0
 if [ $run_scrape = "1" ] ; then
     if [ $# = 0 ] ; then
@@ -106 +107 @@
         && /bin/rm -f "votes-1957-${esc_cutoff_endyear}.csv" \
         && /bin/rm -f "contestants-1957-${esc_cutoff_endyear}.csv" \
-        && python3 -u ./main.py --start 1957 --end $esc_cutoff_endyear \
-               \
-        && /bin/rm -f "votes-1956-${esc_cutoff_endyear}.csv" \
-        && /bin/rm -f "contestants-1956-${esc_cutoff_endyear}.csv" \
-        && python3 -u ./main.py --start 1956 --end $esc_cutoff_endyear \
-        && cd ..
+        && python3 -u ./main.py --start 1957 --end $esc_cutoff_endyear
+    run_scrape_status=$?
+    cd ..
     else
     cd "$prep_dir" \
-        && python3 -u ./main.py $* \
-        && cd ..
+        && python3 -u ./main.py $*
+    run_scrape_status=$?
+    cd ..
     fi
 fi
 
-if [ $? = 0 ] && [ $run_download = "1" ] ; then
-    #wget --no-check-certificate -nc -O "$prep_dir/contestants.csv" \
-    #   https://github.com/Spijkervet/eurovision_dataset/releases/download/2020.0/contestants.csv \
-
+run_download_status=0
+if [ $run_scrape_status = 0 ] && [ $run_download = "1" ] ; then
     if [ $# = 0 ] ; then
     # append CSV fileanme to $*
-    set -- "$@" "contestants-1956-${esc_cutoff_endyear}.csv"
+    set -- "$@" "contestants-1956.csv" "contestants-1957-${esc_cutoff_endyear}.csv"
     fi
     
     cd "$prep_dir" \
-    && python3 -u ./audio.py $* \
-    && cd ..
+    && python3 -u ./audio.py $*
+    run_download_status=$?
+    cd ..
 fi
 
-if [ $? = 0 ] && [ $run_compute = "1" ] ; then
-    if [ $# = 0 ] ; then
-    # append CSV fileanme to $*
-    set -- "$@" "contestants-1956-${esc_cutoff_endyear}.csv"
-    fi
-    
+if [ $run_download_status = 0 ] && [ $run_compute = "1" ] ; then
     cd "$prep_dir" \
     && python3 -u ./audio_features.py $* \