Changeset 32109


Ignore:
Timestamp:
2018-01-18T23:28:54+13:00 (4 years ago)
Author:
davidb
Message:

Changes made after testing through YARN

Location:
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest
Files:
1 added
3 edited

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/JSONLIST-YARN-INGEST.sh

    r32107 r32109  
    11#!/bin/bash
    22
    3 nohup ./scripts/JSONLIST-RUN-YARN-SPARK.sh TMP-faceted-htrc-fictsample-ef20 &
     3nohup ./scripts/JSONLIST-RUN-YARN-SPARK.sh \
     4      pair-tree-annika-1k-fiction-vol-ids.txt TMP-faceted-htrc-fictsample-ef20 &
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/scripts/_RUN.sh

    r32107 r32109  
    2727
    2828    if [ "x$input_dir" != "x" ] ; then
    29     if [ $show_usage = "json" ] ; then
     29    if [ $class_mode = "json" ] ; then
    3030        show_usage=0
    3131    fi
     
    130130    #cmd="$cmd --properties /homea/dbbridge/extracted-features-solr/solr-ingest/ef-solr.properties $seq_file $*"
    131131else
    132     cmd="$cmd --properties ef-solr.properties $json_filelist $input_dir $*"
    133     #cmd="$cmd --properties /homea/dbbridge/extracted-features-solr/solr-ingest/ef-solr.properties $json_filelist $input_dir $*"
     132    cmd="$cmd --properties ef-solr.properties $input_dir $json_filelist $*"
     133    #cmd="$cmd --properties /homea/dbbridge/extracted-features-solr/solr-ingest/ef-solr.properties $input_dir $json_filelist $*"
    134134
    135135fi
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeUtil.java

    r32106 r32109  
    22
    33import java.io.IOException;
     4import java.io.Serializable;
    45import java.util.ArrayList;
    56import java.util.HashMap;
    67import java.util.Iterator;
    7 
     8   
    89import org.apache.hadoop.io.Text;
    910import org.apache.spark.api.java.function.FlatMapFunction;
     
    1415import org.json.JSONObject;
    1516
    16 /*
    17 class PagedJSON implements Function<String, Boolean> {
    18 
     17public class PerVolumeUtil implements Serializable
     18{
    1919    private static final long serialVersionUID = 1L;
    20 
    21     public Boolean call(String s) { return s.contains("a"); }
    22 }
    23  */
    24 
    25 
    26 //public class PerVolumeJSON implements VoidFunction<String>
    27 public class PerVolumeUtil
    28 {
    29     //private static final long serialVersionUID = 1L;
    3020    protected String _input_dir;
    3121    protected String _whitelist_filename;
Note: See TracChangeset for help on using the changeset viewer.