source: gs3-extensions/maori-lang-detection/hdfs-cc-work/conf/nutch-site.xml@ 33596

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
 Licensed to the Apache Software Foundation (ASF) under one or more
 contributor license agreements.  See the NOTICE file distributed with
 this work for additional information regarding copyright ownership.
 The ASF licenses this file to You under the Apache License, Version 2.0
 (the "License"); you may not use this file except in compliance with
 the License.  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>

  <property>
    <name>http.agent.name</name>
    <value>GreenstoneTeam</value>
  </property>

  <property>
    <name>storage.data.store.class</name>
    <value>org.apache.gora.hbase.store.HBaseStore</value>
    <description>Default class for storing data</description>
  </property>
  <!-- GS NOTE:
       Refer to https://lucene.472066.n3.nabble.com/protocol-http-or-protocol-httpclient-td4262473.html
       Plugin protocol-http now supports https, ssl too. Whatever problems may still exist with protocol-httpclient (reco
       mmended below for https, back when it had more problems than at present) also exist in protocol-http.
       In short, there's no need to turn on protocol-httpclient for https, as protocol-http works (equally well) for http
       s pages too.
  -->
  <property>
    <name>plugin.includes</name>
    <value>protocol-http|urlfilter-(regex|validator)|parse-(html|tika)|index-(basic|anchor)|indexer-solr|scoring-opic|url
    normalizer-(pass|regex|basic)</value>
    <description>Regular expression naming plugin directory names to
    include.  Any plugin not matching this expression is excluded.
    In any case you need at least include the nutch-extensionpoints plugin. By
    default Nutch includes crawling just HTML and plain text via HTTP,
    and basic indexing and search plugins. In order to use HTTPS please enable
    protocol-httpclient, but be aware of possible intermittent problems with the
    underlying commons-httpclient library. Set parsefilter-naivebayes for classification based focused crawler.
    </description>
  </property>


  <!-- https://lucene.472066.n3.nabble.com/Content-of-size-X-was-truncated-to-Y-td4003517.html -->
  <!--
      <property>
      <name>parser.skip.truncated</name>
      <value>false</value>
      <description>Boolean value for whether we should skip parsing for truncated documents. By default this
      property is activated due to extremely high levels of CPU which parsing can sometimes take.
      </description>
      </property>
        -->  
  
  <property>
    <name>http.content.limit</name>
    <value>-1</value>
    <description>The length limit for downloaded content using the http://
    protocol, in bytes. If this value is nonnegative (>=0), content longer
    than it will be truncated; otherwise, no truncation at all. Do not
    confuse this setting with the file.content.limit setting.
    </description>
  </property>

  <!--https://stackoverflow.com/questions/4871972/how-to-speed-up-crawling-in-nutch-->
  <!--
      <property>
      <name>fetcher.threads.per.queue</name>
      <value>1</value>
      <description>This number is the maximum number of threads that
      should be allowed to access a queue at one time. Setting it to
      a value > 1 will cause the Crawl-Delay value from robots.txt to
      be ignored and the value of fetcher.server.min.delay to be used
      as a delay between successive requests to the same server instead
      of fetcher.server.delay.
      </description>
      </property>
  -->
  <property>
    <name>generate.max.count</name>
    <value>50</value>
    <description>The maximum number of urls in a single
    fetchlist.  -1 if unlimited. The urls are counted according
    to the value of the parameter generator.count.mode.
    </description>
  </property>
  <property>
    <name>fetcher.throughput.threshold.pages</name>
    <value>1</value>
    <description>The threshold of minimum pages per second. If the fetcher downloads less
    pages per second than the configured threshold, the fetcher stops, preventing slow queue's
    from stalling the throughput. This threshold must be an integer. This can be useful when
    fetcher.timelimit.mins is hard to determine. The default value of -1 disables this check.
    </description>
  </property>
  <property>
    <name>fetcher.server.delay</name>
    <value>0.5</value>
    <description>The number of seconds the fetcher will delay between
    successive requests to the same server. Note that this might get
    overriden by a Crawl-Delay from a robots.txt and is used ONLY if
    fetcher.threads.per.queue is set to 1.
    </description>
  </property>
  

</configuration>