NUTCH-3011 HttpRobotRulesParser: handle HTTP 429 Too Many Requests sa…

…me as server errors (HTTP 5xx)

conf/nutch-default.xml

@@ -141,25 +141,26 @@
   <name>http.robots.503.defer.visits</name>
   <value>true</value>
   <description>Temporarily suspend fetching from a host if the
-  robots.txt response is HTTP 503 or any other 5xx server error. See
-  also http.robots.503.defer.visits.delay and
+  robots.txt response is HTTP 503 or any other 5xx server error
+  and HTTP 429 Too Many Requests. See also
+  http.robots.503.defer.visits.delay and
   http.robots.503.defer.visits.retries</description>
 </property>
 
 <property>
   <name>http.robots.503.defer.visits.delay</name>
   <value>300000</value>
   <description>Time in milliseconds to suspend crawling a host if the
-  robots.txt response is HTTP 5xx - see
+  robots.txt response is HTTP 5xx or 429 Too Many Requests - see
   http.robots.503.defer.visits.</description>
 </property>
 
 <property>
   <name>http.robots.503.defer.visits.retries</name>
   <value>3</value>
   <description>Number of retries crawling a host if the robots.txt
-  response is HTTP 5xx - see http.robots.503.defer.visits. After n
-  retries the host queue is dropped for this segment/cycle.
+  response is HTTP 5xx or 429 - see http.robots.503.defer.visits.
+  After n retries the host queue is dropped for this segment/cycle.
   </description>
 </property>
 

src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java

@@ -229,7 +229,8 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url,
         else if ((code == 403) && (!allowForbidden))
           robotRules = FORBID_ALL_RULES; // use forbid all
 
-        else if (code >= 500) {
+        else if (code >= 500 || code == 429) {
+          // 5xx server errors or 429 Too Many Requests
           cacheRule = false; // try again later to fetch robots.txt
           if (deferVisits503) {
             // signal fetcher to suspend crawling for this host