Changeset 30975 for other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java
- Timestamp:
- 2016-10-28T14:28:51+13:00 (8 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java
r30951 r30975 15 15 protected String _input_dir; 16 16 protected String _json_list_filename; 17 protected String _solr_url; 17 18 protected String _output_dir; 18 19 19 20 protected int _verbosity; 20 21 21 public PrepareForIngest(String input_dir, String json_list_filename, String output_dir, int verbosity) 22 public PrepareForIngest(String input_dir, String json_list_filename, 23 String solr_url, String output_dir, int verbosity) 22 24 { 23 25 _input_dir = input_dir; … … 42 44 JavaRDD<String> json_list_data = jsc.textFile(_json_list_filename,NUM_PARTITIONS).cache(); 43 45 44 PagedJSON paged_json = new PagedJSON(_input_dir,_ output_dir,_verbosity);46 PagedJSON paged_json = new PagedJSON(_input_dir,_solr_url, _output_dir,_verbosity); 45 47 JavaRDD<String> json_ids = json_list_data.flatMap(paged_json).cache(); 46 48 … … 52 54 System.out.println(""); 53 55 54 System.out.println("############");55 56 String rdd_save_file = "rdd-solr-json-page-files"; 56 57 json_ids.saveAsTextFile(rdd_save_file); 58 System.out.println("############"); 57 59 System.out.println("# Saved RDD of Solr JSON page files, top-level, as:"); 58 60 System.out.println("# " + rdd_save_file); … … 63 65 } 64 66 67 public static void print_usage(HelpFormatter formatter, Options options) 68 { 69 formatter.printHelp("RUN.bash [options] input-dir json-filelist.txt", options); 70 } 65 71 public static void main(String[] args) { 66 72 … … 86 92 options.addOption(verbosity_opt); 87 93 88 //CommandLineParser parser = new DefaultParser(); // 1.3 and above 94 Option output_dir_opt = new Option("o", "output-dir", true, 95 "If specified, save BZipped Solr JSON files to this directory"); 96 output_dir_opt.setRequired(false); 97 options.addOption(output_dir_opt); 98 99 Option solr_url_opt = new Option("u", "solr-url", true, 100 "If specified, the URL to post the Solr JSON data to"); 101 solr_url_opt.setRequired(false); 102 options.addOption(solr_url_opt); 103 104 Option dry_run_opt = new Option("r", "dry-run", false, 105 "Used to initiate a 'dry-run' where the files are all read in, but nothing is ingested/saved"); 106 dry_run_opt.setRequired(false); 107 options.addOption(dry_run_opt); 89 108 90 109 // need to work with CLI v1.2 as this is the JAR that is bundled with Hadoop/Spark 91 110 CommandLineParser parser = new GnuParser(); 111 //CommandLineParser parser = new DefaultParser(); // if working with CLI v1.3 and above 112 92 113 HelpFormatter formatter = new HelpFormatter(); 93 CommandLine cmd ;114 CommandLine cmd = null; 94 115 95 116 try { … … 98 119 catch (ParseException e) { 99 120 System.err.println(e.getMessage()); 100 formatter.printHelp("RUN.bash [options] json-file-list.txt input-dir output-dir",options);121 print_usage(formatter,options); 101 122 System.exit(1); 102 return;123 //return; // prevents 'cmd may not be assigned' compiler error in Eclipse 103 124 } 104 125 … … 111 132 int verbosity = Integer.parseInt(verbosity_str); 112 133 134 String output_dir = cmd.getOptionValue("output-dir",null); 135 String solr_url = cmd.getOptionValue("solr-url",null); 136 boolean dry_run = cmd.hasOption("dry-run"); 137 113 138 String[] filtered_args = cmd.getArgs(); 114 139 115 if (filtered_args.length != 3) {116 formatter.printHelp("RUN.bash [options] json-filelist.txt input-dir output-dir",options);140 if (filtered_args.length != 2) { 141 print_usage(formatter,options); 117 142 System.exit(1); 143 } 144 145 if (!dry_run && ((output_dir == null) && (solr_url==null))) { 146 System.err.println("Need to specify either --solr-url or --output-dir otherwise generated files are not ingested/saved"); 147 print_usage(formatter,options); 118 148 } 119 149 String json_list_filename = filtered_args[0]; 120 150 String input_dir = filtered_args[1]; 121 String output_dir = filtered_args[2];151 //String output_dir = filtered_args[2]; 122 152 123 PrepareForIngest prep_for_ingest = new PrepareForIngest(input_dir,json_list_filename,output_dir,verbosity); 153 PrepareForIngest prep_for_ingest 154 = new PrepareForIngest(input_dir,json_list_filename,solr_url,output_dir,verbosity); 124 155 prep_for_ingest.exec(); 125 156
Note:
See TracChangeset
for help on using the changeset viewer.