Changeset 30975 for other-projects
- Timestamp:
- 2016-10-28T14:28:51+13:00 (7 years ago)
- Location:
- other-projects/hathitrust/solr-extracted-features/trunk
- Files:
-
- 4 edited
Legend:
- Unmodified
- Added
- Removed
-
other-projects/hathitrust/solr-extracted-features/trunk/RUN-PD-CLUSTER.bash
r30929 r30975 7 7 #input_dir="hdfs://10.10.0.52:9000/user/htrc/pd-ef-json-files" 8 8 9 output_dir=hdfs://master:9000/user/htrc/pd-solr-json-files10 9 #output_dir=hdfs://master:9000/user/htrc/pd-solr-json-files 10 solr_url="http://10.11.0.53:8983/solr/htrc-pd-ef/update" 11 11 12 12 master_opt="--master spark://10.10.0.52:7077" -
other-projects/hathitrust/solr-extracted-features/trunk/_RUN.bash
r30952 r30975 19 19 fi 20 20 21 if [ "x$output_dir" = "x" ] ; then22 echo "_RUN.bash: Failed to set 'output_dir'" 1>&223 exit24 fi21 #if [ "x$output_dir" = "x" ] ; then 22 # echo "_RUN.bash: Failed to set 'output_dir'" 1>&2 23 # exit 24 #fi 25 25 26 26 run_jps=0 … … 71 71 72 72 self_contained_jar=target/htrc-ef-ingest-0.9-jar-with-dependencies.jar 73 base_cmd="spark-submit --class org.hathitrust.PrepareForIngest $master_opt $self_contained_jar"73 cmd="spark-submit --class org.hathitrust.PrepareForIngest $master_opt $self_contained_jar" 74 74 75 cmd="$base_cmd --verbosity 1 $json_filelist $input_dir $output_dir $*" 75 if [ "x$solr_url" != "x" ] ; then 76 cmd="$cmd --solr-url $solr_url" 77 fi 78 79 if [ "x$output_dir" != "x" ] ; then 80 cmd="$cmd --output-dir $output_dir" 81 fi 82 83 84 cmd="$cmd --verbosity 1 $input_dir $json_filelist $*" 76 85 77 86 echo "****" … … 79 88 echo "* $cmd" 80 89 echo "****" 90 81 91 if [ "$run_jps" = "1" ] ; then 82 92 echo "* Monitor progress on Spark cluster through:" -
other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java
r30974 r30975 31 31 32 32 protected String _input_dir; 33 protected String _solr_url; 33 34 protected String _output_dir; 34 35 protected int _verbosity; 35 36 36 public PagedJSON(String input_dir, String output_dir, int verbosity)37 public PagedJSON(String input_dir, String solr_url, String output_dir, int verbosity) 37 38 { 38 39 _input_dir = input_dir; 40 _solr_url = solr_url; 39 41 _output_dir = output_dir; 40 42 _verbosity = verbosity; … … 170 172 */ 171 173 172 //return solr_doc_json;173 174 return solr_update_json; 174 175 } … … 288 289 */ 289 290 290 saveSolrDoc(solr_add_doc_json,output_json_bz2); 291 //postSolrDoc(solr_add_doc_json); 292 291 if (_solr_url != null) { 292 postSolrDoc(solr_add_doc_json); 293 } 294 295 if (_output_dir != null) { 296 saveSolrDoc(solr_add_doc_json,output_json_bz2); 297 } 293 298 } 294 299 else { -
other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java
r30951 r30975 15 15 protected String _input_dir; 16 16 protected String _json_list_filename; 17 protected String _solr_url; 17 18 protected String _output_dir; 18 19 19 20 protected int _verbosity; 20 21 21 public PrepareForIngest(String input_dir, String json_list_filename, String output_dir, int verbosity) 22 public PrepareForIngest(String input_dir, String json_list_filename, 23 String solr_url, String output_dir, int verbosity) 22 24 { 23 25 _input_dir = input_dir; … … 42 44 JavaRDD<String> json_list_data = jsc.textFile(_json_list_filename,NUM_PARTITIONS).cache(); 43 45 44 PagedJSON paged_json = new PagedJSON(_input_dir,_ output_dir,_verbosity);46 PagedJSON paged_json = new PagedJSON(_input_dir,_solr_url, _output_dir,_verbosity); 45 47 JavaRDD<String> json_ids = json_list_data.flatMap(paged_json).cache(); 46 48 … … 52 54 System.out.println(""); 53 55 54 System.out.println("############");55 56 String rdd_save_file = "rdd-solr-json-page-files"; 56 57 json_ids.saveAsTextFile(rdd_save_file); 58 System.out.println("############"); 57 59 System.out.println("# Saved RDD of Solr JSON page files, top-level, as:"); 58 60 System.out.println("# " + rdd_save_file); … … 63 65 } 64 66 67 public static void print_usage(HelpFormatter formatter, Options options) 68 { 69 formatter.printHelp("RUN.bash [options] input-dir json-filelist.txt", options); 70 } 65 71 public static void main(String[] args) { 66 72 … … 86 92 options.addOption(verbosity_opt); 87 93 88 //CommandLineParser parser = new DefaultParser(); // 1.3 and above 94 Option output_dir_opt = new Option("o", "output-dir", true, 95 "If specified, save BZipped Solr JSON files to this directory"); 96 output_dir_opt.setRequired(false); 97 options.addOption(output_dir_opt); 98 99 Option solr_url_opt = new Option("u", "solr-url", true, 100 "If specified, the URL to post the Solr JSON data to"); 101 solr_url_opt.setRequired(false); 102 options.addOption(solr_url_opt); 103 104 Option dry_run_opt = new Option("r", "dry-run", false, 105 "Used to initiate a 'dry-run' where the files are all read in, but nothing is ingested/saved"); 106 dry_run_opt.setRequired(false); 107 options.addOption(dry_run_opt); 89 108 90 109 // need to work with CLI v1.2 as this is the JAR that is bundled with Hadoop/Spark 91 110 CommandLineParser parser = new GnuParser(); 111 //CommandLineParser parser = new DefaultParser(); // if working with CLI v1.3 and above 112 92 113 HelpFormatter formatter = new HelpFormatter(); 93 CommandLine cmd ;114 CommandLine cmd = null; 94 115 95 116 try { … … 98 119 catch (ParseException e) { 99 120 System.err.println(e.getMessage()); 100 formatter.printHelp("RUN.bash [options] json-file-list.txt input-dir output-dir",options);121 print_usage(formatter,options); 101 122 System.exit(1); 102 return;123 //return; // prevents 'cmd may not be assigned' compiler error in Eclipse 103 124 } 104 125 … … 111 132 int verbosity = Integer.parseInt(verbosity_str); 112 133 134 String output_dir = cmd.getOptionValue("output-dir",null); 135 String solr_url = cmd.getOptionValue("solr-url",null); 136 boolean dry_run = cmd.hasOption("dry-run"); 137 113 138 String[] filtered_args = cmd.getArgs(); 114 139 115 if (filtered_args.length != 3) {116 formatter.printHelp("RUN.bash [options] json-filelist.txt input-dir output-dir",options);140 if (filtered_args.length != 2) { 141 print_usage(formatter,options); 117 142 System.exit(1); 143 } 144 145 if (!dry_run && ((output_dir == null) && (solr_url==null))) { 146 System.err.println("Need to specify either --solr-url or --output-dir otherwise generated files are not ingested/saved"); 147 print_usage(formatter,options); 118 148 } 119 149 String json_list_filename = filtered_args[0]; 120 150 String input_dir = filtered_args[1]; 121 String output_dir = filtered_args[2];151 //String output_dir = filtered_args[2]; 122 152 123 PrepareForIngest prep_for_ingest = new PrepareForIngest(input_dir,json_list_filename,output_dir,verbosity); 153 PrepareForIngest prep_for_ingest 154 = new PrepareForIngest(input_dir,json_list_filename,solr_url,output_dir,verbosity); 124 155 prep_for_ingest.exec(); 125 156
Note:
See TracChangeset
for help on using the changeset viewer.