Changeset 30941 for other-projects
- Timestamp:
- 2016-10-26T14:16:25+13:00 (7 years ago)
- Location:
- other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/ClusterFileIO.java
r30940 r30941 25 25 public class ClusterFileIO { 26 26 27 pr otected static FileSystem_fs = null;27 private static FileSystem __fs = null; 28 28 29 /* 29 30 public static void initOLD(JavaSparkContext java_spark_context) 30 31 { … … 37 38 } 38 39 } 40 */ 39 41 42 /* 40 43 public static void init(String input_dir) 41 44 { … … 53 56 54 57 } 58 */ 59 60 public static FileSystem getFileSystemInstance(String input_dir) 61 { 62 if (__fs == null) { 63 64 try { 65 Configuration conf = new Configuration(); 66 URI uri = new URI(input_dir); 67 __fs = FileSystem.get(uri, conf); 68 } 69 catch (URISyntaxException e) { 70 e.printStackTrace(); 71 } 72 catch (IOException e) { 73 e.printStackTrace(); 74 } 75 } 76 return __fs; 77 } 78 55 79 public static boolean isHDFS(String fileIn) 56 80 { … … 60 84 protected static boolean exists(String file) 61 85 { 86 FileSystem fs = getFileSystemInstance(file); 87 62 88 //Configuration conf = jsc.hadoopConfiguration(); 63 89 //FileSystem fs = org.apache.hadoop.fs.FileSystem.get(conf); … … 65 91 66 92 try { 67 exists = _fs.exists(new Path(file));93 exists = fs.exists(new Path(file)); 68 94 } catch (IllegalArgumentException e) { 69 95 exists = false; … … 77 103 throws IOException 78 104 { 105 FileSystem fs = getFileSystemInstance(fileIn); 106 79 107 BufferedInputStream bis = null; 80 108 … … 86 114 87 115 //FSDataInputStream fin = _fs.open(new Path(fileIn)); 88 FSDataInputStream fin = _fs.open(new Path(uri));116 FSDataInputStream fin = fs.open(new Path(uri)); 89 117 90 118 bis = new BufferedInputStream(fin); … … 101 129 */ 102 130 103 FSDataInputStream fin = _fs.open(new Path(fileIn));131 FSDataInputStream fin = fs.open(new Path(fileIn)); 104 132 105 133 bis = new BufferedInputStream(fin); -
other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java
r30937 r30941 36 36 { 37 37 String spark_app_name = "HathiTrust Extract Features: Prepare for Solr Ingest"; 38 spark_app_name += " [" + _json_list_filename + "]";38 spark_app_name += " [" + _json_list_filename + "]"; 39 39 40 40 SparkConf conf = new SparkConf().setAppName(spark_app_name); 41 41 JavaSparkContext jsc = new JavaSparkContext(conf); 42 ClusterFileIO.init(_input_dir);42 //ClusterFileIO.init(_input_dir); 43 43 44 44 // Check output directory exists, and create it if not
Note:
See TracChangeset
for help on using the changeset viewer.