Mirrors/awesome-rust
2
0
Fork 0
mirror of https://github.com/rust-unofficial/awesome-rust synced 2024-11-15 00:27:39 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #1106 from palfrey/too-many-requests

Browse source 
Checker: Don't retry 429's and don't worry about them if we've got a previous success
This commit is contained in:
Tom Parker-Shemilt 2021-06-26 19:41:34 +01:00 committed by GitHub
commit 4ff49d1345
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 16 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
src/main.rs
View file

@ -26,6 +26,9 @@ enum CheckerError {
location: Option<String>,
},
#[fail(display = "too many requests")]
TooManyRequests,
#[fail(display = "reqwest error: {}", error)]
ReqwestError {
error: String,
@ -191,6 +194,12 @@ fn get_url_core(url: String) -> BoxFuture<'static, (String, Result<(), CheckerEr
return (url, res);
}
if status == StatusCode::TOO_MANY_REQUESTS {
// We get a lot of these, and we should not retry as they'll just fail again
warn!("Error while getting {}: {}", url, status);
return (url, Err(CheckerError::TooManyRequests));
}
warn!("Error while getting {}, retrying: {}", url, status);
if status.is_redirection() {
res = Err(CheckerError::HttpError {status: status.as_u16(), location: ok.headers().get(header::LOCATION).and_then(|h| h.to_str().ok()).map(|x| x.to_string())});
@ -365,6 +374,13 @@ async fn main() -> Result<(), Error> {
failed +=1;
continue;
}
CheckerError::TooManyRequests => {
// too many tries
if link.last_working.is_some() {
info!("Ignoring 429 failure on {} as we've seen success before", url);
continue;
}
}
_ => {}
};
if let Some(last_working) = link.last_working {