source: other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/scripts/FULL-DOWNLOAD-EVERY-N.sh@ 31062

Last change on this file since 31062 was 31062, checked in by davidb, 6 years ago

Added in -W option so check-sum calculation is skipped

  • Property svn:executable set to *
File size: 733 bytes
Line 
1#!/bin/bash
2
3argc=$#
4
5if [ "$argc" != "1" ] && [ "$argc" != "2" ] ; then
6 echo "Usage FULL-DOWNLOAD-EVERY-N.sh <num> [output-dir]" >&2
7 exit 1
8fi
9
10num=$1
11
12if [ "$argc" = "2" ] ; then
13 output_dir="$2"
14else
15 output_dir="full-ef-json-files"
16fi
17
18# Remove any trailing /.
19output_dir="${output_dir%/\.}"
20output_dir="${output_dir%/}"
21
22if [ ! -d "$output_dir" ] ; then
23 echo ""
24 echo "****"
25 echo "* Creating $output_dir"
26 echo "****"
27 echo ""
28 mkdir "$output_dir"
29fi
30
31# Now ensure a trailing /. is present!
32output_dir="${output_dir}/."
33
34echo ""
35echo "Processing input file: full-listing-step${num}.txt"
36echo ""
37
38rsync -W -pav --files-from=full-listing-step${num}.txt \
39 data.analytics.hathitrust.org::features/ $output_dir
Note: See TracBrowser for help on using the repository browser.