Changeset 32109

Show
Ignore:
Timestamp:
18.01.2018 23:28:54 (4 months ago)
Author:
davidb
Message:

Changes made after testing through YARN

Location:
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest
Files:
1 added
3 modified

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/JSONLIST-YARN-INGEST.sh

    r32107 r32109  
    11#!/bin/bash 
    22 
    3 nohup ./scripts/JSONLIST-RUN-YARN-SPARK.sh TMP-faceted-htrc-fictsample-ef20 & 
     3nohup ./scripts/JSONLIST-RUN-YARN-SPARK.sh \ 
     4      pair-tree-annika-1k-fiction-vol-ids.txt TMP-faceted-htrc-fictsample-ef20 & 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/scripts/_RUN.sh

    r32107 r32109  
    2727 
    2828    if [ "x$input_dir" != "x" ] ; then 
    29     if [ $show_usage = "json" ] ; then 
     29    if [ $class_mode = "json" ] ; then 
    3030        show_usage=0 
    3131    fi 
     
    130130    #cmd="$cmd --properties /homea/dbbridge/extracted-features-solr/solr-ingest/ef-solr.properties $seq_file $*" 
    131131else 
    132     cmd="$cmd --properties ef-solr.properties $json_filelist $input_dir $*" 
    133     #cmd="$cmd --properties /homea/dbbridge/extracted-features-solr/solr-ingest/ef-solr.properties $json_filelist $input_dir $*" 
     132    cmd="$cmd --properties ef-solr.properties $input_dir $json_filelist $*" 
     133    #cmd="$cmd --properties /homea/dbbridge/extracted-features-solr/solr-ingest/ef-solr.properties $input_dir $json_filelist $*" 
    134134 
    135135fi 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeUtil.java

    r32106 r32109  
    22 
    33import java.io.IOException; 
     4import java.io.Serializable; 
    45import java.util.ArrayList; 
    56import java.util.HashMap; 
    67import java.util.Iterator; 
    7  
     8     
    89import org.apache.hadoop.io.Text; 
    910import org.apache.spark.api.java.function.FlatMapFunction; 
     
    1415import org.json.JSONObject; 
    1516 
    16 /* 
    17 class PagedJSON implements Function<String, Boolean> { 
    18  
     17public class PerVolumeUtil implements Serializable 
     18{ 
    1919    private static final long serialVersionUID = 1L; 
    20  
    21     public Boolean call(String s) { return s.contains("a"); } 
    22 } 
    23  */ 
    24  
    25  
    26 //public class PerVolumeJSON implements VoidFunction<String>  
    27 public class PerVolumeUtil 
    28 { 
    29     //private static final long serialVersionUID = 1L; 
    3020    protected String _input_dir; 
    3121    protected String _whitelist_filename;