Merge pull request #302 from tycho/icon-cache-ttl

implement TTLs for icon cache

.env

@@ -10,6 +10,11 @@
 # ICON_CACHE_FOLDER=data/icon_cache
 # ATTACHMENTS_FOLDER=data/attachments
 
+## Cache time-to-live for successfully obtained icons, in seconds (0 is "forever")
+# ICON_CACHE_TTL=2592000
+## Cache time-to-live for icons which weren't available, in seconds (0 is "forever")
+# ICON_CACHE_NEGTTL=259200
+
 ## Web vault settings
 # WEB_VAULT_FOLDER=web-vault/
 # WEB_VAULT_ENABLED=true

src/api/icons.rs

@@ -1,5 +1,7 @@
 use std::io::prelude::*;
-use std::fs::{create_dir_all, File};
+use std::fs::{symlink_metadata, create_dir_all, remove_file, File};
+use std::time::SystemTime;
+use std::error::Error;
 
 use rocket::Route;
 use rocket::response::Content;
@@ -44,12 +46,23 @@ fn get_icon (domain: &str) -> Vec<u8> {
         },
         Err(e) => {
             error!("Error downloading icon: {:?}", e);
+            mark_negcache(&path);
             get_fallback_icon()
         }
     }
 }
 
 fn get_cached_icon(path: &str) -> Option<Vec<u8>> {
+    // Check for expiration of negatively cached copy
+    if icon_is_negcached(path) {
+        return Some(get_fallback_icon());
+    }
+
+    // Check for expiration of successfully cached copy
+    if icon_is_expired(path) {
+        return None
+    }
+
     // Try to read the cached icon, and return it if it exists
     if let Ok(mut f) = File::open(path) {
         let mut buffer = Vec::new();
@@ -62,6 +75,47 @@ fn get_cached_icon(path: &str) -> Option<Vec<u8>> {
     None
 }
 
+fn file_is_expired(path: &str, ttl: u64) -> Result<bool, Box<Error>> {
+    let meta = symlink_metadata(path)?;
+    let modified = meta.modified()?;
+    let age = SystemTime::now().duration_since(modified)?;
+
+    Ok(ttl > 0 && ttl <= age.as_secs())
+}
+
+fn icon_is_negcached(path: &str) -> bool {
+    let miss_indicator = path.to_owned() + ".miss";
+    let expired = file_is_expired(&miss_indicator, CONFIG.icon_cache_negttl);
+    match expired {
+        // No longer negatively cached, drop the marker
+        Ok(true) => {
+            match remove_file(&miss_indicator) {
+                Ok(_) => {},
+                Err(e) => {
+                    error!("Could not remove negative cache indicator for icon {:?}: {:?}", path, e);
+                }
+            }
+            false
+        },
+
+        // The marker hasn't expired yet.
+        Ok(false) => { true }
+
+        // The marker is missing or inaccessible in some way.
+        Err(_) => { false }
+    }
+}
+
+fn mark_negcache(path: &str) {
+    let miss_indicator = path.to_owned() + ".miss";
+    File::create(&miss_indicator).expect("Error creating negative cache marker");
+}
+
+fn icon_is_expired(path: &str) -> bool {
+    let expired = file_is_expired(path, CONFIG.icon_cache_ttl);
+    expired.unwrap_or(true)
+}
+
 fn get_icon_url(domain: &str) -> String {
     if CONFIG.local_icon_extractor {
         format!("http://{}/favicon.ico", domain)

src/main.rs

@@ -255,6 +255,9 @@ pub struct Config {
     icon_cache_folder: String,
     attachments_folder: String,
 
+    icon_cache_ttl: u64,
+    icon_cache_negttl: u64,
+
     private_rsa_key: String,
     private_rsa_key_pem: String,
     public_rsa_key: String,
@@ -304,6 +307,11 @@ impl Config {
             icon_cache_folder: get_env_or("ICON_CACHE_FOLDER", format!("{}/{}", &df, "icon_cache")),
             attachments_folder: get_env_or("ATTACHMENTS_FOLDER", format!("{}/{}", &df, "attachments")),
 
+            // icon_cache_ttl defaults to 30 days (30 * 24 * 60 * 60 seconds)
+            icon_cache_ttl: get_env_or("ICON_CACHE_TTL", 2592000u64),
+            // icon_cache_negttl defaults to 3 days (3 * 24 * 60 * 60 seconds)
+            icon_cache_negttl: get_env_or("ICON_CACHE_NEGTTL", 259200u64),
+
             private_rsa_key: format!("{}.der", &key),
             private_rsa_key_pem: format!("{}.pem", &key),
             public_rsa_key: format!("{}.pub.der", &key),