Index: gs3-extensions/maori-lang-detection/hdfs-cc-work/conf/nutch-site.xml
===================================================================
--- gs3-extensions/maori-lang-detection/hdfs-cc-work/conf/nutch-site.xml	(revision 33596)
+++ gs3-extensions/maori-lang-detection/hdfs-cc-work/conf/nutch-site.xml	(revision 33596)
@@ -0,0 +1,119 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+  <property>
+    <name>http.agent.name</name>
+    <value>GreenstoneTeam</value>
+  </property>
+
+  <property>
+    <name>storage.data.store.class</name>
+    <value>org.apache.gora.hbase.store.HBaseStore</value>
+    <description>Default class for storing data</description>
+  </property>
+  <!-- GS NOTE:
+       Refer to https://lucene.472066.n3.nabble.com/protocol-http-or-protocol-httpclient-td4262473.html
+       Plugin protocol-http now supports https, ssl too. Whatever problems may still exist with protocol-httpclient (reco
+       mmended below for https, back when it had more problems than at present) also exist in protocol-http.
+       In short, there's no need to turn on protocol-httpclient for https, as protocol-http works (equally well) for http
+       s pages too.
+  -->
+  <property>
+    <name>plugin.includes</name>
+    <value>protocol-http|urlfilter-(regex|validator)|parse-(html|tika)|index-(basic|anchor)|indexer-solr|scoring-opic|url
+    normalizer-(pass|regex|basic)</value>
+    <description>Regular expression naming plugin directory names to
+    include.  Any plugin not matching this expression is excluded.
+    In any case you need at least include the nutch-extensionpoints plugin. By
+    default Nutch includes crawling just HTML and plain text via HTTP,
+    and basic indexing and search plugins. In order to use HTTPS please enable
+    protocol-httpclient, but be aware of possible intermittent problems with the
+    underlying commons-httpclient library. Set parsefilter-naivebayes for classification based focused crawler.
+    </description>
+  </property>
+
+
+  <!-- https://lucene.472066.n3.nabble.com/Content-of-size-X-was-truncated-to-Y-td4003517.html -->
+  <!--
+      <property>
+      <name>parser.skip.truncated</name>
+      <value>false</value>
+      <description>Boolean value for whether we should skip parsing for truncated documents. By default this
+      property is activated due to extremely high levels of CPU which parsing can sometimes take.
+      </description>
+      </property>
+        -->  
+  
+  <property>
+    <name>http.content.limit</name>
+    <value>-1</value>
+    <description>The length limit for downloaded content using the http://
+    protocol, in bytes. If this value is nonnegative (>=0), content longer
+    than it will be truncated; otherwise, no truncation at all. Do not
+    confuse this setting with the file.content.limit setting.
+    </description>
+  </property>
+
+  <!--https://stackoverflow.com/questions/4871972/how-to-speed-up-crawling-in-nutch-->
+  <!--
+      <property>
+      <name>fetcher.threads.per.queue</name>
+      <value>1</value>
+      <description>This number is the maximum number of threads that
+      should be allowed to access a queue at one time. Setting it to
+      a value > 1 will cause the Crawl-Delay value from robots.txt to
+      be ignored and the value of fetcher.server.min.delay to be used
+      as a delay between successive requests to the same server instead
+      of fetcher.server.delay.
+      </description>
+      </property>
+  -->
+  <property>
+    <name>generate.max.count</name>
+    <value>50</value>
+    <description>The maximum number of urls in a single
+    fetchlist.  -1 if unlimited. The urls are counted according
+    to the value of the parameter generator.count.mode.
+    </description>
+  </property>
+  <property>
+    <name>fetcher.throughput.threshold.pages</name>
+    <value>1</value>
+    <description>The threshold of minimum pages per second. If the fetcher downloads less
+    pages per second than the configured threshold, the fetcher stops, preventing slow queue's
+    from stalling the throughput. This threshold must be an integer. This can be useful when
+    fetcher.timelimit.mins is hard to determine. The default value of -1 disables this check.
+    </description>
+  </property>
+  <property>
+    <name>fetcher.server.delay</name>
+    <value>0.5</value>
+    <description>The number of seconds the fetcher will delay between
+    successive requests to the same server. Note that this might get
+    overriden by a Crawl-Delay from a robots.txt and is used ONLY if
+    fetcher.threads.per.queue is set to 1.
+    </description>
+  </property>
+  
+
+</configuration>
Index: gs3-extensions/maori-lang-detection/hdfs-cc-work/conf/regex-urlfilter.GS_TEMPLATE
===================================================================
--- gs3-extensions/maori-lang-detection/hdfs-cc-work/conf/regex-urlfilter.GS_TEMPLATE	(revision 33596)
+++ gs3-extensions/maori-lang-detection/hdfs-cc-work/conf/regex-urlfilter.GS_TEMPLATE	(revision 33596)
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# The default url filter.
+# Better for whole-internet crawling.
+
+# Each non-comment, non-blank line contains a regular expression
+# prefixed by '+' or '-'.  The first matching pattern in the file
+# determines whether a URL is included or ignored.  If no pattern
+# matches, the URL is ignored.
+
+# skip file: ftp: and mailto: urls
+-^(file|ftp|mailto):
+
+# skip image and other suffixes we can't yet parse
+# for a more extensive coverage use the urlfilter-suffix plugin
+-\.(gif|GIF|jpg|JPG|png|PNG|ico|ICO|css|CSS|sit|SIT|eps|EPS|wmf|WMF|zip|ZIP|ppt|PPT|mpg|MPG|xls|XLS|gz|GZ|rpm|RPM|tgz|TGZ|mov|MOV|exe|EXE|jpeg|JPEG|bmp|BMP|js|JS)$
+
+# skip URLs containing certain characters as probable queries, etc.
+-[?*!@=]
+
+# skip URLs with slash-delimited segment that repeats 3+ times, to break loops
+-.*(/[^/]+)/[^/]+\1/[^/]+\1/
+
+# accept anything else
+#+.
+
+# batchcrawl.sh will automatically append regex url filters below for each site to be crawled
