source: other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/scripts/SERIAL-INGEST.sh@ 32102

#!/bin/bash

m2home=C:\\Users\\davidb\\.m2\\repository

jars=D:\\cygwin64\\home\\davidb\\research\\code-managed\\hathitrust\\wcsa\\extracted-features-solr\\trunk\\solr-ingest\\target\\classes\;$m2home\\org\\apache\\spark\\spark-core_2.11\\2.0.1\\spark-core_2.11-2.0.1.jar\;$m2home\\org\\apache\\avro\\avro-mapred\\1.7.7\\avro-mapred-1.7.7-hadoop2.jar\;$m2home\\org\\apache\\avro\\avro-ipc\\1.7.7\\avro-ipc-1.7.7.jar\;$m2home\\org\\apache\\avro\\avro\\1.7.7\\avro-1.7.7.jar\;$m2home\\org\\apache\\avro\\avro-ipc\\1.7.7\\avro-ipc-1.7.7-tests.jar\;$m2home\\org\\codehaus\\jackson\\jackson-core-asl\\1.9.13\\jackson-core-asl-1.9.13.jar\;$m2home\\org\\codehaus\\jackson\\jackson-mapper-asl\\1.9.13\\jackson-mapper-asl-1.9.13.jar\;$m2home\\com\\twitter\\chill_2.11\\0.8.0\\chill_2.11-0.8.0.jar\;$m2home\\com\\esotericsoftware\\kryo-shaded\\3.0.3\\kryo-shaded-3.0.3.jar\;$m2home\\com\\esotericsoftware\\minlog\\1.3.0\\minlog-1.3.0.jar\;$m2home\\org\\objenesis\\objenesis\\2.1\\objenesis-2.1.jar\;$m2home\\com\\twitter\\chill-java\\0.8.0\\chill-java-0.8.0.jar\;$m2home\\org\\apache\\xbean\\xbean-asm5-shaded\\4.4\\xbean-asm5-shaded-4.4.jar\;$m2home\\org\\apache\\hadoop\\hadoop-client\\2.2.0\\hadoop-client-2.2.0.jar\;$m2home\\org\\apache\\hadoop\\hadoop-common\\2.2.0\\hadoop-common-2.2.0.jar\;$m2home\\org\\apache\\commons\\commons-math\\2.1\\commons-math-2.1.jar\;$m2home\\xmlenc\\xmlenc\\0.52\\xmlenc-0.52.jar\;$m2home\\commons-io\\commons-io\\2.1\\commons-io-2.1.jar\;$m2home\\commons-lang\\commons-lang\\2.5\\commons-lang-2.5.jar\;$m2home\\commons-configuration\\commons-configuration\\1.6\\commons-configuration-1.6.jar\;$m2home\\commons-collections\\commons-collections\\3.2.1\\commons-collections-3.2.1.jar\;$m2home\\commons-digester\\commons-digester\\1.8\\commons-digester-1.8.jar\;$m2home\\commons-beanutils\\commons-beanutils\\1.7.0\\commons-beanutils-1.7.0.jar\;$m2home\\commons-beanutils\\commons-beanutils-core\\1.8.0\\commons-beanutils-core-1.8.0.jar\;$m2home\\com\\google\\protobuf\\protobuf-java\\2.5.0\\protobuf-java-2.5.0.jar\;$m2home\\org\\apache\\hadoop\\hadoop-auth\\2.2.0\\hadoop-auth-2.2.0.jar\;$m2home\\org\\apache\\hadoop\\hadoop-hdfs\\2.2.0\\hadoop-hdfs-2.2.0.jar\;$m2home\\org\\mortbay\\jetty\\jetty-util\\6.1.26\\jetty-util-6.1.26.jar\;$m2home\\org\\apache\\hadoop\\hadoop-mapreduce-client-app\\2.2.0\\hadoop-mapreduce-client-app-2.2.0.jar\;$m2home\\org\\apache\\hadoop\\hadoop-mapreduce-client-common\\2.2.0\\hadoop-mapreduce-client-common-2.2.0.jar\;$m2home\\org\\apache\\hadoop\\hadoop-yarn-client\\2.2.0\\hadoop-yarn-client-2.2.0.jar\;$m2home\\com\\google\\inject\\guice\\3.0\\guice-3.0.jar\;$m2home\\javax\\inject\\javax.inject\\1\\javax.inject-1.jar\;$m2home\\aopalliance\\aopalliance\\1.0\\aopalliance-1.0.jar\;$m2home\\org\\apache\\hadoop\\hadoop-yarn-server-common\\2.2.0\\hadoop-yarn-server-common-2.2.0.jar\;$m2home\\org\\apache\\hadoop\\hadoop-mapreduce-client-shuffle\\2.2.0\\hadoop-mapreduce-client-shuffle-2.2.0.jar\;$m2home\\org\\apache\\hadoop\\hadoop-yarn-api\\2.2.0\\hadoop-yarn-api-2.2.0.jar\;$m2home\\org\\apache\\hadoop\\hadoop-mapreduce-client-core\\2.2.0\\hadoop-mapreduce-client-core-2.2.0.jar\;$m2home\\org\\apache\\hadoop\\hadoop-yarn-common\\2.2.0\\hadoop-yarn-common-2.2.0.jar\;$m2home\\org\\apache\\hadoop\\hadoop-mapreduce-client-jobclient\\2.2.0\\hadoop-mapreduce-client-jobclient-2.2.0.jar\;$m2home\\org\\apache\\hadoop\\hadoop-annotations\\2.2.0\\hadoop-annotations-2.2.0.jar\;$m2home\\org\\apache\\spark\\spark-launcher_2.11\\2.0.1\\spark-launcher_2.11-2.0.1.jar\;$m2home\\org\\apache\\spark\\spark-network-common_2.11\\2.0.1\\spark-network-common_2.11-2.0.1.jar\;$m2home\\org\\fusesource\\leveldbjni\\leveldbjni-all\\1.8\\leveldbjni-all-1.8.jar\;$m2home\\com\\fasterxml\\jackson\\core\\jackson-annotations\\2.6.5\\jackson-annotations-2.6.5.jar\;$m2home\\org\\apache\\spark\\spark-network-shuffle_2.11\\2.0.1\\spark-network-shuffle_2.11-2.0.1.jar\;$m2home\\org\\apache\\spark\\spark-unsafe_2.11\\2.0.1\\spark-unsafe_2.11-2.0.1.jar\;$m2home\\net\\java\\dev\\jets3t\\jets3t\\0.7.1\\jets3t-0.7.1.jar\;$m2home\\commons-codec\\commons-codec\\1.3\\commons-codec-1.3.jar\;$m2home\\commons-httpclient\\commons-httpclient\\3.1\\commons-httpclient-3.1.jar\;$m2home\\org\\apache\\curator\\curator-recipes\\2.4.0\\curator-recipes-2.4.0.jar\;$m2home\\org\\apache\\curator\\curator-framework\\2.4.0\\curator-framework-2.4.0.jar\;$m2home\\org\\apache\\curator\\curator-client\\2.4.0\\curator-client-2.4.0.jar\;$m2home\\org\\apache\\zookeeper\\zookeeper\\3.4.5\\zookeeper-3.4.5.jar\;$m2home\\javax\\servlet\\javax.servlet-api\\3.1.0\\javax.servlet-api-3.1.0.jar\;$m2home\\org\\apache\\commons\\commons-lang3\\3.3.2\\commons-lang3-3.3.2.jar\;$m2home\\org\\apache\\commons\\commons-math3\\3.4.1\\commons-math3-3.4.1.jar\;$m2home\\com\\google\\code\\findbugs\\jsr305\\1.3.9\\jsr305-1.3.9.jar\;$m2home\\org\\slf4j\\slf4j-api\\1.7.16\\slf4j-api-1.7.16.jar\;$m2home\\org\\slf4j\\jul-to-slf4j\\1.7.16\\jul-to-slf4j-1.7.16.jar\;$m2home\\org\\slf4j\\jcl-over-slf4j\\1.7.16\\jcl-over-slf4j-1.7.16.jar\;$m2home\\log4j\\log4j\\1.2.17\\log4j-1.2.17.jar\;$m2home\\org\\slf4j\\slf4j-log4j12\\1.7.16\\slf4j-log4j12-1.7.16.jar\;$m2home\\com\\ning\\compress-lzf\\1.0.3\\compress-lzf-1.0.3.jar\;$m2home\\org\\xerial\\snappy\\snappy-java\\1.1.2.6\\snappy-java-1.1.2.6.jar\;$m2home\\net\\jpountz\\lz4\\lz4\\1.3.0\\lz4-1.3.0.jar\;$m2home\\org\\roaringbitmap\\RoaringBitmap\\0.5.11\\RoaringBitmap-0.5.11.jar\;$m2home\\commons-net\\commons-net\\2.2\\commons-net-2.2.jar\;$m2home\\org\\json4s\\json4s-jackson_2.11\\3.2.11\\json4s-jackson_2.11-3.2.11.jar\;$m2home\\org\\json4s\\json4s-core_2.11\\3.2.11\\json4s-core_2.11-3.2.11.jar\;$m2home\\org\\json4s\\json4s-ast_2.11\\3.2.11\\json4s-ast_2.11-3.2.11.jar\;$m2home\\com\\thoughtworks\\paranamer\\paranamer\\2.6\\paranamer-2.6.jar\;$m2home\\org\\scala-lang\\scalap\\2.11.0\\scalap-2.11.0.jar\;$m2home\\org\\scala-lang\\scala-compiler\\2.11.0\\scala-compiler-2.11.0.jar\;$m2home\\org\\scala-lang\\modules\\scala-parser-combinators_2.11\\1.0.1\\scala-parser-combinators_2.11-1.0.1.jar\;$m2home\\org\\glassfish\\jersey\\core\\jersey-client\\2.22.2\\jersey-client-2.22.2.jar\;$m2home\\javax\\ws\\rs\\javax.ws.rs-api\\2.0.1\\javax.ws.rs-api-2.0.1.jar\;$m2home\\org\\glassfish\\hk2\\hk2-api\\2.4.0-b34\\hk2-api-2.4.0-b34.jar\;$m2home\\org\\glassfish\\hk2\\hk2-utils\\2.4.0-b34\\hk2-utils-2.4.0-b34.jar\;$m2home\\org\\glassfish\\hk2\\external\\aopalliance-repackaged\\2.4.0-b34\\aopalliance-repackaged-2.4.0-b34.jar\;$m2home\\org\\glassfish\\hk2\\external\\javax.inject\\2.4.0-b34\\javax.inject-2.4.0-b34.jar\;$m2home\\org\\glassfish\\hk2\\hk2-locator\\2.4.0-b34\\hk2-locator-2.4.0-b34.jar\;$m2home\\org\\javassist\\javassist\\3.18.1-GA\\javassist-3.18.1-GA.jar\;$m2home\\org\\glassfish\\jersey\\core\\jersey-common\\2.22.2\\jersey-common-2.22.2.jar\;$m2home\\javax\\annotation\\javax.annotation-api\\1.2\\javax.annotation-api-1.2.jar\;$m2home\\org\\glassfish\\jersey\\bundles\\repackaged\\jersey-guava\\2.22.2\\jersey-guava-2.22.2.jar\;$m2home\\org\\glassfish\\hk2\\osgi-resource-locator\\1.0.1\\osgi-resource-locator-1.0.1.jar\;$m2home\\org\\glassfish\\jersey\\core\\jersey-server\\2.22.2\\jersey-server-2.22.2.jar\;$m2home\\org\\glassfish\\jersey\\media\\jersey-media-jaxb\\2.22.2\\jersey-media-jaxb-2.22.2.jar\;$m2home\\javax\\validation\\validation-api\\1.1.0.Final\\validation-api-1.1.0.Final.jar\;$m2home\\org\\glassfish\\jersey\\containers\\jersey-container-servlet\\2.22.2\\jersey-container-servlet-2.22.2.jar\;$m2home\\org\\glassfish\\jersey\\containers\\jersey-container-servlet-core\\2.22.2\\jersey-container-servlet-core-2.22.2.jar\;$m2home\\org\\apache\\mesos\\mesos\\0.21.1\\mesos-0.21.1-shaded-protobuf.jar\;$m2home\\io\\netty\\netty-all\\4.0.29.Final\\netty-all-4.0.29.Final.jar\;$m2home\\io\\netty\\netty\\3.8.0.Final\\netty-3.8.0.Final.jar\;$m2home\\com\\clearspring\\analytics\\stream\\2.7.0\\stream-2.7.0.jar\;$m2home\\io\\dropwizard\\metrics\\metrics-core\\3.1.2\\metrics-core-3.1.2.jar\;$m2home\\io\\dropwizard\\metrics\\metrics-jvm\\3.1.2\\metrics-jvm-3.1.2.jar\;$m2home\\io\\dropwizard\\metrics\\metrics-json\\3.1.2\\metrics-json-3.1.2.jar\;$m2home\\io\\dropwizard\\metrics\\metrics-graphite\\3.1.2\\metrics-graphite-3.1.2.jar\;$m2home\\com\\fasterxml\\jackson\\core\\jackson-databind\\2.6.5\\jackson-databind-2.6.5.jar\;$m2home\\com\\fasterxml\\jackson\\core\\jackson-core\\2.6.5\\jackson-core-2.6.5.jar\;$m2home\\com\\fasterxml\\jackson\\module\\jackson-module-scala_2.11\\2.6.5\\jackson-module-scala_2.11-2.6.5.jar\;$m2home\\org\\scala-lang\\scala-reflect\\2.11.7\\scala-reflect-2.11.7.jar\;$m2home\\com\\fasterxml\\jackson\\module\\jackson-module-paranamer\\2.6.5\\jackson-module-paranamer-2.6.5.jar\;$m2home\\org\\apache\\ivy\\ivy\\2.4.0\\ivy-2.4.0.jar\;$m2home\\oro\\oro\\2.0.8\\oro-2.0.8.jar\;$m2home\\net\\razorvine\\pyrolite\\4.9\\pyrolite-4.9.jar\;$m2home\\net\\sf\\py4j\\py4j\\0.10.3\\py4j-0.10.3.jar\;$m2home\\org\\apache\\spark\\spark-tags_2.11\\2.0.1\\spark-tags_2.11-2.0.1.jar\;$m2home\\org\\scalatest\\scalatest_2.11\\2.2.6\\scalatest_2.11-2.2.6.jar\;$m2home\\org\\scala-lang\\modules\\scala-xml_2.11\\1.0.2\\scala-xml_2.11-1.0.2.jar\;$m2home\\org\\spark-project\\spark\\unused\\1.0.0\\unused-1.0.0.jar\;$m2home\\org\\scala-lang\\scala-library\\2.12.1\\scala-library-2.12.1.jar\;$m2home\\org\\mongodb\\mongo-java-driver\\3.4.1\\mongo-java-driver-3.4.1.jar\;$m2home\\org\\apache\\commons\\commons-compress\\1.10\\commons-compress-1.10.jar\;$m2home\\org\\json\\json\\20160212\\json-20160212.jar\;$m2home\\commons-cli\\commons-cli\\1.2\\commons-cli-1.2.jar\;$m2home\\com\\google\\guava\\guava\\20.0\\guava-20.0.jar\;$m2home\\org\\apache\\lucene\\lucene-analyzers-icu\\6.3.0\\lucene-analyzers-icu-6.3.0.jar\;$m2home\\org\\apache\\lucene\\lucene-analyzers-common\\6.3.0\\lucene-analyzers-common-6.3.0.jar\;$m2home\\org\\apache\\lucene\\lucene-core\\6.3.0\\lucene-core-6.3.0.jar\;$m2home\\com\\ibm\\icu\\icu4j\\56.1\\icu4j-56.1.jar

#java -cp target\\htrc-ef-ingest-0.9-jar-with-dependencies.jar\;hadoop-common-2.7.3.jar\;spark-core_2.11-2.0.1.jar\;commons-lang3-3.4.jar\;commons-logging-1.1.3.jar\;guava-20.0.jar\;commons-collections-3.2.2.jar\;hadoop-nfs-2.7.3.jar\;commons-configuration-1.6.jar\;commons-lang-2.6.jar\;hadoop-mapreduce-client-core-2.7.3.jar \

java -cp target\\htrc-ef-ingest-0.9-jar-with-dependencies.jar\;$jars \
     org.hathitrust.extractedfeatures.ProcessForSerialSolrIngest \
     --properties serial-ef-solr.properties \
     --solr-base-url http://solr1.ischool.illinois.edu:8983/solr \
    annika\\pair-tree-annika-1k-fiction-vol-ids.txt \
  faceted-htrc-fictsample-ef20

# --read-only \

# --solr-base-url http://solr1-s:8983/solr
#    --properties /homea/dbbridge/extracted-features-solr/solr-ingest/ef-solr.properties hdfs:///user/capitanu/data/packed-ef faceted-htrc-full-ef20