Changeset 32963

Timestamp:

2019-04-01T17:45:14+13:00 (5 years ago)

Author:

davidb

Message:

Added text and some refinement of scripts to make things easier to run

Location:

other-projects/is-sheet-music-encore/trunk/gen-corpus-ids

Files:

• HATHI-EXTRACT-FORMAT.sh (modified) (1 diff)
• HATHI-EXTRACT-PD-NON-GOOGLE.sh (modified) (1 diff)
• README.txt (added)

other-projects/is-sheet-music-encore/trunk/gen-corpus-ids/HATHI-EXTRACT-FORMAT.sh

@@ -2 +2 @@
 
 input=${1:-'hathi_full_20190301.txt.gz'}
+output=${2:-'hathi_brief_20190301.txt'}
 
+echo ""
+echo "===="
+echo "  Script to extract Format (and related fields, such as copyright)"
+echo "  from HathiTrust tab-delimited metadata dump"
+echo "===="
+
+echo ""
+echo "Reading in  : $input"
+echo "Writing out : $output"
+echo ""
+
+echo "Processing ..."
 zcat "$input" \
-     | awk -F '\t' '{print $1 "\t" $3 "\t" $20 "\t" $24} '
+     | awk -F '\t' '{print $1 "\t" $3 "\t" $20 "\t" $24} ' \
+     > "$output"
+
+echo "... Done"
+echo ""

other-projects/is-sheet-music-encore/trunk/gen-corpus-ids/HATHI-EXTRACT-PD-NON-GOOGLE.sh

@@ -2 +2 @@
 
 input=${1:-'hathi_brief_20190301.txt'}
+output=${2:-'hathi_pd_MU_Not-Google_20190301.txt'}
+
+echo ""
+echo "===="
+echo "  Script to filter down the extracted Music Format data entries that at"
+echo "  publicly available: public domain and NOT scanned by Google"
+echo "===="
+
+echo ""
+echo "Reading in  : $input"
+echo "Writing out : $output"
+echo ""
+
+echo "Processing ..."
 
 cat "$input" \
-    | awk -F '\t' '$2 == "pd" && $3 == "MU" && $4 != "google" { print $0 }' 
+    | awk -F '\t' '$2 == "pd" && $3 == "MU" && $4 != "google" { print $0 }' \
+    > "$output"
 
-#  grep MU | grep -v google 
+echo "... Done"
+echo ""
+