From 7bc2d0bb567acd3ce06af643a181c44fb15f0c29 Mon Sep 17 00:00:00 2001
From: Tom Parker-Shemilt <palfrey@tevp.net>
Date: Thu, 24 Jun 2021 16:28:08 +0100
Subject: [PATCH] Don't retry 429's and don't worry about them if we've got a
 previous success

---
 src/main.rs | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/main.rs b/src/main.rs
index e0324ce..8861ca3 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -26,6 +26,9 @@ enum CheckerError {
         location: Option<String>,
     },
 
+    #[fail(display = "too many requests")]
+    TooManyRequests,    
+
     #[fail(display = "reqwest error: {}", error)]
     ReqwestError {
         error: String,
@@ -191,6 +194,12 @@ fn get_url_core(url: String) -> BoxFuture<'static, (String, Result<(), CheckerEr
                             return (url, res);
                         }
 
+                        if status == StatusCode::TOO_MANY_REQUESTS {
+                            // We get a lot of these, and we should not retry as they'll just fail again
+                            warn!("Error while getting {}: {}", url, status);
+                            return (url, Err(CheckerError::TooManyRequests));
+                        }
+
                         warn!("Error while getting {}, retrying: {}", url, status);
                         if status.is_redirection() {
                             res = Err(CheckerError::HttpError {status: status.as_u16(), location: ok.headers().get(header::LOCATION).and_then(|h| h.to_str().ok()).map(|x| x.to_string())});
@@ -365,6 +374,13 @@ async fn main() -> Result<(), Error> {
                     failed +=1;
                     continue;
                 }
+                CheckerError::TooManyRequests => {
+                    // too many tries
+                    if link.last_working.is_some() {
+                        info!("Ignoring 429 failure on {} as we've seen success before", url);
+                        continue;
+                    }
+                }
                 _ => {}
             };
             if let Some(last_working) = link.last_working {