Add support for bookmark thumbnails (#721)

* Preview Image

* fix tests

* add test

* download preview image

* relative path

* gst

* details view

* fix tests

* Improve preview image styles

* Remove preview image URL from model

* Revert form changes

* update tests

* make it work in uwsgi

---------

Co-authored-by: Sascha Ißbrücker <sascha.issbruecker@gmail.com>

.gitignore

@@ -194,3 +194,5 @@ typings/
 # ublock + chromium
 /uBlock0.chromium
 /chromium-profile
+# direnv
+/.direnv

bookmarks/api/routes.py

@@ -99,7 +99,10 @@ class BookmarkViewSet(
         # Either return metadata from existing bookmark, or scrape from URL
         if bookmark:
             metadata = WebsiteMetadata(
-                url, bookmark.website_title, bookmark.website_description
+                url,
+                bookmark.website_title,
+                bookmark.website_description,
+                None,
             )
         else:
             metadata = website_loader.load_website_metadata(url)

bookmarks/migrations/0034_bookmark_preview_image_file_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.0.3 on 2024-05-07 07:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookmarks", "0033_userprofile_default_mark_unread"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="bookmark",
+            name="preview_image_file",
+            field=models.CharField(blank=True, max_length=512, null=True),
+        ),
+        migrations.AddField(
+            model_name="userprofile",
+            name="enable_preview_images",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookmarks/models.py

@@ -59,6 +59,7 @@ class Bookmark(models.Model):
     website_description = models.TextField(blank=True, null=True)
     web_archive_snapshot_url = models.CharField(max_length=2048, blank=True)
     favicon_file = models.CharField(max_length=512, blank=True)
+    preview_image_file = models.CharField(max_length=512, blank=True, null=True)
     unread = models.BooleanField(default=False)
     is_archived = models.BooleanField(default=False)
     shared = models.BooleanField(default=False)
@@ -394,6 +395,7 @@ class UserProfile(models.Model):
     enable_sharing = models.BooleanField(default=False, null=False)
     enable_public_sharing = models.BooleanField(default=False, null=False)
     enable_favicons = models.BooleanField(default=False, null=False)
+    enable_preview_images = models.BooleanField(default=False, null=False)
     display_url = models.BooleanField(default=False, null=False)
     display_view_bookmark_action = models.BooleanField(default=True, null=False)
     display_edit_bookmark_action = models.BooleanField(default=True, null=False)
@@ -420,6 +422,7 @@ class UserProfileForm(forms.ModelForm):
             "enable_sharing",
             "enable_public_sharing",
             "enable_favicons",
+            "enable_preview_images",
             "enable_automatic_html_snapshots",
             "display_url",
             "display_view_bookmark_action",

bookmarks/services/bookmarks.py

@@ -40,6 +40,8 @@ def create_bookmark(bookmark: Bookmark, tag_string: str, current_user: User):
     tasks.create_web_archive_snapshot(current_user, bookmark, False)
     # Load favicon
     tasks.load_favicon(current_user, bookmark)
+    # Load preview image
+    tasks.load_preview_image(current_user, bookmark)
     # Create HTML snapshot
     if current_user.profile.enable_automatic_html_snapshots:
         tasks.create_html_snapshot(bookmark)
@@ -58,6 +60,8 @@ def update_bookmark(bookmark: Bookmark, tag_string, current_user: User):
     bookmark.save()
     # Update favicon
     tasks.load_favicon(current_user, bookmark)
+    # Update preview image
+    tasks.load_preview_image(current_user, bookmark)
 
     if has_url_changed:
         # Update web archive snapshot, if URL changed

bookmarks/services/preview_image_loader.py

@@ -0,0 +1,46 @@
+import logging
+import mimetypes
+import os.path
+import hashlib
+from pathlib import Path
+
+import requests
+from django.conf import settings
+from bookmarks.services import website_loader
+
+logger = logging.getLogger(__name__)
+
+
+def _ensure_preview_folder():
+    Path(settings.LD_PREVIEW_FOLDER).mkdir(parents=True, exist_ok=True)
+
+
+def _url_to_filename(preview_image: str) -> str:
+    return hashlib.md5(preview_image.encode()).hexdigest()
+
+
+def _get_image_path(preview_image_file: str) -> Path:
+    return Path(os.path.join(settings.LD_PREVIEW_FOLDER, preview_image_file))
+
+
+def load_preview_image(url: str) -> str | None:
+    _ensure_preview_folder()
+
+    metadata = website_loader.load_website_metadata(url)
+    if not metadata.preview_image:
+        logger.debug(f"Could not find preview image in metadata: {url}")
+        return None
+
+    logger.debug(f"Loading preview image: {metadata.preview_image}")
+    with requests.get(metadata.preview_image, stream=True) as response:
+        content_type = response.headers["Content-Type"]
+        preview_image_hash = _url_to_filename(url)
+        file_extension = mimetypes.guess_extension(content_type)
+        preview_image_file = f"{preview_image_hash}{file_extension}"
+        preview_image_path = _get_image_path(preview_image_file)
+        with open(preview_image_path, "wb") as file:
+            for chunk in response.iter_content(chunk_size=8192):
+                file.write(chunk)
+    logger.debug(f"Saved preview image as: {preview_image_path}")
+
+    return preview_image_file

bookmarks/services/tasks.py

@@ -15,7 +15,7 @@ from waybackpy.exceptions import WaybackError, TooManyRequestsError, NoCDXRecord
 
 import bookmarks.services.wayback
 from bookmarks.models import Bookmark, BookmarkAsset, UserProfile
-from bookmarks.services import favicon_loader, singlefile
+from bookmarks.services import favicon_loader, singlefile, preview_image_loader
 from bookmarks.services.website_loader import DEFAULT_USER_AGENT
 
 logger = logging.getLogger(__name__)
@@ -221,6 +221,30 @@ def _schedule_refresh_favicons_task(user_id: int):
         _load_favicon_task(bookmark.id)
 
 
+def load_preview_image(user: User, bookmark: Bookmark):
+    if user.profile.enable_preview_images and not settings.LD_DISABLE_BACKGROUND_TASKS:
+        _load_preview_image_task(bookmark.id)
+
+
+@task()
+def _load_preview_image_task(bookmark_id: int):
+    try:
+        bookmark = Bookmark.objects.get(id=bookmark_id)
+    except Bookmark.DoesNotExist:
+        return
+
+    logger.info(f"Load preview image for bookmark. url={bookmark.url}")
+
+    new_preview_image_file = preview_image_loader.load_preview_image(bookmark.url)
+
+    if new_preview_image_file != bookmark.preview_image_file:
+        bookmark.preview_image_file = new_preview_image_file
+        bookmark.save(update_fields=["preview_image_file"])
+        logger.info(
+            f"Successfully updated preview image for bookmark. url={bookmark.url} preview_image_file={new_preview_image_file}"
+        )
+
+
 def is_html_snapshot_feature_active() -> bool:
     return settings.LD_ENABLE_SNAPSHOTS and not settings.LD_DISABLE_BACKGROUND_TASKS
 

bookmarks/services/website_loader.py

@@ -1,6 +1,7 @@
 import logging
 from dataclasses import dataclass
 from functools import lru_cache
+from urllib.parse import urljoin
 
 import requests
 from bs4 import BeautifulSoup
@@ -15,12 +16,14 @@ class WebsiteMetadata:
     url: str
     title: str
     description: str
+    preview_image: str | None
 
     def to_dict(self):
         return {
             "url": self.url,
             "title": self.title,
             "description": self.description,
+            "preview_image": self.preview_image,
         }
 
 
@@ -30,6 +33,7 @@ class WebsiteMetadata:
 def load_website_metadata(url: str):
     title = None
     description = None
+    preview_image = None
     try:
         start = timezone.now()
         page_text = load_page(url)
@@ -55,10 +59,21 @@ def load_website_metadata(url: str):
                 else None
             )
 
+        image_tag = soup.find("meta", attrs={"property": "og:image"})
+        preview_image = image_tag["content"].strip() if image_tag else None
+        if (
+            preview_image
+            and not preview_image.startswith("http://")
+            and not preview_image.startswith("https://")
+        ):
+            preview_image = urljoin(url, preview_image)
+
         end = timezone.now()
         logger.debug(f"Parsing duration: {end - start}")
     finally:
-        return WebsiteMetadata(url=url, title=title, description=description)
+        return WebsiteMetadata(
+            url=url, title=title, description=description, preview_image=preview_image
+        )
 
 
 CHUNK_SIZE = 50 * 1024

bookmarks/styles/bookmark-details.scss

@@ -33,6 +33,15 @@
     text-overflow: ellipsis;
   }
 
+  .preview-image {
+    margin: $unit-4 0;
+
+    img {
+      max-width: 100%;
+      max-height: 200px;
+    }
+  }
+
   dl {
     margin-bottom: 0;
   }

bookmarks/styles/bookmark-page.scss

@@ -128,8 +128,25 @@ ul.bookmark-list {
 /* Bookmarks */
 li[ld-bookmark-item] {
   position: relative;
+  display: flex;
+  gap: $unit-2;
   margin-top: $unit-2;
 
+  .content {
+    flex: 1 1 0;
+    min-width: 0;
+  }
+
+  img.preview-image {
+    flex: 0 0 auto;
+    width: 100px;
+    height: 60px;
+    margin-top: $unit-h;
+    object-fit: cover;
+    border-radius: $border-radius;
+    border: solid 1px $border-color-dark;
+  }
+
   .form-checkbox.bulk-edit-checkbox {
     display: none;
   }
@@ -346,7 +363,7 @@ $bulk-edit-transition-duration: 400ms;
     transition: all $bulk-edit-transition-duration;
 
     .form-icon {
-        top: 0;
+      top: 0;
     }
   }
 

bookmarks/templates/bookmarks/bookmark_list.html

@@ -10,138 +10,143 @@
       data-bookmarks-total="{{ bookmark_list.bookmarks_total }}">
     {% for bookmark_item in bookmark_list.items %}
       <li ld-bookmark-item{% if bookmark_item.css_classes %} class="{{ bookmark_item.css_classes }}"{% endif %}>
-        <div class="title">
+        <div class="content">
-          <label class="form-checkbox bulk-edit-checkbox">
+          <div class="title">
-            <input type="checkbox" name="bookmark_id" value="{{ bookmark_item.id }}">
+            <label class="form-checkbox bulk-edit-checkbox">
-            <i class="form-icon"></i>
+              <input type="checkbox" name="bookmark_id" value="{{ bookmark_item.id }}">
-          </label>
+              <i class="form-icon"></i>
-          {% if bookmark_item.favicon_file and bookmark_list.show_favicons %}
+            </label>
-            <img src="{% static bookmark_item.favicon_file %}" alt="">
+            {% if bookmark_item.favicon_file and bookmark_list.show_favicons %}
-          {% endif %}
+              <img class="favicon" src="{% static bookmark_item.favicon_file %}" alt="">
-          <a href="{{ bookmark_item.url }}" target="{{ bookmark_list.link_target }}" rel="noopener">
+            {% endif %}
-            <span>{{ bookmark_item.title }}</span>
+            <a href="{{ bookmark_item.url }}" target="{{ bookmark_list.link_target }}" rel="noopener">
-          </a>
+              <span>{{ bookmark_item.title }}</span>
-        </div>
-        {% if bookmark_list.show_url %}
-          <div class="url-path truncate">
-            <a href="{{ bookmark_item.url }}" target="{{ bookmark_list.link_target }}" rel="noopener"
-               class="url-display">
-              {{ bookmark_item.url }}
             </a>
           </div>
-        {% endif %}
+          {% if bookmark_list.show_url %}
-        {% if bookmark_list.description_display == 'inline' %}
+            <div class="url-path truncate">
-          <div class="description inline truncate">
+              <a href="{{ bookmark_item.url }}" target="{{ bookmark_list.link_target }}" rel="noopener"
+                class="url-display">
+                {{ bookmark_item.url }}
+              </a>
+            </div>
+          {% endif %}
+          {% if bookmark_list.description_display == 'inline' %}
+            <div class="description inline truncate">
+              {% if bookmark_item.tag_names %}
+                <span class="tags">
+                  {% for tag_name in bookmark_item.tag_names %}
+                    <a href="?{% add_tag_to_query tag_name %}">{{ tag_name|hash_tag }}</a>
+                  {% endfor %}
+                </span>
+              {% endif %}
+              {% if bookmark_item.tag_names and bookmark_item.description %} | {% endif %}
+              {% if bookmark_item.description %}
+                <span>{{ bookmark_item.description }}</span>
+              {% endif %}
+            </div>
+          {% else %}
+            {% if bookmark_item.description %}
+              <div class="description separate">{{ bookmark_item.description }}</div>
+            {% endif %}
             {% if bookmark_item.tag_names %}
-              <span class="tags">
+              <div class="tags">
                 {% for tag_name in bookmark_item.tag_names %}
                   <a href="?{% add_tag_to_query tag_name %}">{{ tag_name|hash_tag }}</a>
                 {% endfor %}
+              </div>
+            {% endif %}
+          {% endif %}
+          {% if bookmark_item.notes %}
+            <div class="notes bg-gray text-gray-dark">
+              <div class="markdown">{% markdown bookmark_item.notes %}</div>
+            </div>
+          {% endif %}
+          <div class="actions text-gray">
+            {% if bookmark_item.display_date %}
+              {% if bookmark_item.web_archive_snapshot_url %}
+                <a href="{{ bookmark_item.web_archive_snapshot_url }}"
+                  title="Show snapshot on the Internet Archive Wayback Machine"
+                  target="{{ bookmark_list.link_target }}"
+                  rel="noopener">
+                  {{ bookmark_item.display_date }} ∞
+                </a>
+              {% else %}
+                <span>{{ bookmark_item.display_date }}</span>
+              {% endif %}
+              <span>|</span>
+            {% endif %}
+            {# View link is visible for both owned and shared bookmarks #}
+            {% if bookmark_list.show_view_action %}
+              <a ld-fetch="{% url 'bookmarks:details_modal' bookmark_item.id %}?return_url={{ bookmark_list.return_url|urlencode }}"
+                ld-on="click" ld-target="body|append"
+                href="{% url 'bookmarks:details' bookmark_item.id %}">View</a>
+            {% endif %}
+            {% if bookmark_item.is_editable %}
+              {# Bookmark owner actions #}
+              {% if bookmark_list.show_edit_action %}
+                <a href="{% url 'bookmarks:edit' bookmark_item.id %}?return_url={{ bookmark_list.return_url|urlencode }}">Edit</a>
+              {% endif %}
+              {% if bookmark_list.show_archive_action %}
+                {% if bookmark_item.is_archived %}
+                  <button type="submit" name="unarchive" value="{{ bookmark_item.id }}"
+                          class="btn btn-link btn-sm">Unarchive
+                  </button>
+                {% else %}
+                  <button type="submit" name="archive" value="{{ bookmark_item.id }}"
+                          class="btn btn-link btn-sm">Archive
+                  </button>
+                {% endif %}
+              {% endif %}
+              {% if bookmark_list.show_remove_action %}
+                <button ld-confirm-button type="submit" name="remove" value="{{ bookmark_item.id }}"
+                        class="btn btn-link btn-sm">Remove
+                </button>
+              {% endif %}
+            {% else %}
+              {# Shared bookmark actions #}
+              <span>Shared by
+                <a href="?{% replace_query_param user=bookmark_item.owner.username %}">{{ bookmark_item.owner.username }}</a>
               </span>
             {% endif %}
-            {% if bookmark_item.tag_names and bookmark_item.description %} | {% endif %}
+            {% if bookmark_item.has_extra_actions %}
-            {% if bookmark_item.description %}
+              <div class="extra-actions">
-              <span>{{ bookmark_item.description }}</span>
+                <span class="hide-sm">|</span>
+                {% if bookmark_item.show_mark_as_read %}
+                  <button type="submit" name="mark_as_read" value="{{ bookmark_item.id }}"
+                          class="btn btn-link btn-sm btn-icon"
+                          ld-confirm-button ld-confirm-icon="ld-icon-read" ld-confirm-question="Mark as read?">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16">
+                      <use xlink:href="#ld-icon-unread"></use>
+                    </svg>
+                    Unread
+                  </button>
+                {% endif %}
+                {% if bookmark_item.show_unshare %}
+                  <button type="submit" name="unshare" value="{{ bookmark_item.id }}"
+                          class="btn btn-link btn-sm btn-icon"
+                          ld-confirm-button ld-confirm-icon="ld-icon-unshare" ld-confirm-question="Unshare?">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16">
+                      <use xlink:href="#ld-icon-share"></use>
+                    </svg>
+                    Shared
+                  </button>
+                {% endif %}
+                {% if bookmark_item.show_notes_button %}
+                  <button type="button" class="btn btn-link btn-sm btn-icon toggle-notes">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16">
+                      <use xlink:href="#ld-icon-note"></use>
+                    </svg>
+                    Notes
+                  </button>
+                {% endif %}
+              </div>
             {% endif %}
           </div>
-        {% else %}
-          {% if bookmark_item.description %}
-            <div class="description separate">{{ bookmark_item.description }}</div>
-          {% endif %}
-          {% if bookmark_item.tag_names %}
-            <div class="tags">
-              {% for tag_name in bookmark_item.tag_names %}
-                <a href="?{% add_tag_to_query tag_name %}">{{ tag_name|hash_tag }}</a>
-              {% endfor %}
-            </div>
-          {% endif %}
-        {% endif %}
-        {% if bookmark_item.notes %}
-          <div class="notes bg-gray text-gray-dark">
-            <div class="markdown">{% markdown bookmark_item.notes %}</div>
-          </div>
-        {% endif %}
-        <div class="actions text-gray">
-          {% if bookmark_item.display_date %}
-            {% if bookmark_item.web_archive_snapshot_url %}
-              <a href="{{ bookmark_item.web_archive_snapshot_url }}"
-                 title="Show snapshot on the Internet Archive Wayback Machine"
-                 target="{{ bookmark_list.link_target }}"
-                 rel="noopener">
-                {{ bookmark_item.display_date }} ∞
-              </a>
-            {% else %}
-              <span>{{ bookmark_item.display_date }}</span>
-            {% endif %}
-            <span>|</span>
-          {% endif %}
-          {# View link is visible for both owned and shared bookmarks #}
-          {% if bookmark_list.show_view_action %}
-            <a ld-fetch="{% url 'bookmarks:details_modal' bookmark_item.id %}?return_url={{ bookmark_list.return_url|urlencode }}"
-               ld-on="click" ld-target="body|append"
-               href="{% url 'bookmarks:details' bookmark_item.id %}">View</a>
-          {% endif %}
-          {% if bookmark_item.is_editable %}
-            {# Bookmark owner actions #}
-            {% if bookmark_list.show_edit_action %}
-              <a href="{% url 'bookmarks:edit' bookmark_item.id %}?return_url={{ bookmark_list.return_url|urlencode }}">Edit</a>
-            {% endif %}
-            {% if bookmark_list.show_archive_action %}
-              {% if bookmark_item.is_archived %}
-                <button type="submit" name="unarchive" value="{{ bookmark_item.id }}"
-                        class="btn btn-link btn-sm">Unarchive
-                </button>
-              {% else %}
-                <button type="submit" name="archive" value="{{ bookmark_item.id }}"
-                        class="btn btn-link btn-sm">Archive
-                </button>
-              {% endif %}
-            {% endif %}
-            {% if bookmark_list.show_remove_action %}
-              <button ld-confirm-button type="submit" name="remove" value="{{ bookmark_item.id }}"
-                      class="btn btn-link btn-sm">Remove
-              </button>
-            {% endif %}
-          {% else %}
-            {# Shared bookmark actions #}
-            <span>Shared by
-              <a href="?{% replace_query_param user=bookmark_item.owner.username %}">{{ bookmark_item.owner.username }}</a>
-            </span>
-          {% endif %}
-          {% if bookmark_item.has_extra_actions %}
-            <div class="extra-actions">
-              <span class="hide-sm">|</span>
-              {% if bookmark_item.show_mark_as_read %}
-                <button type="submit" name="mark_as_read" value="{{ bookmark_item.id }}"
-                        class="btn btn-link btn-sm btn-icon"
-                        ld-confirm-button ld-confirm-icon="ld-icon-read" ld-confirm-question="Mark as read?">
-                  <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16">
-                    <use xlink:href="#ld-icon-unread"></use>
-                  </svg>
-                  Unread
-                </button>
-              {% endif %}
-              {% if bookmark_item.show_unshare %}
-                <button type="submit" name="unshare" value="{{ bookmark_item.id }}"
-                        class="btn btn-link btn-sm btn-icon"
-                        ld-confirm-button ld-confirm-icon="ld-icon-unshare" ld-confirm-question="Unshare?">
-                  <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16">
-                    <use xlink:href="#ld-icon-share"></use>
-                  </svg>
-                  Shared
-                </button>
-              {% endif %}
-              {% if bookmark_item.show_notes_button %}
-                <button type="button" class="btn btn-link btn-sm btn-icon toggle-notes">
-                  <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16">
-                    <use xlink:href="#ld-icon-note"></use>
-                  </svg>
-                  Notes
-                </button>
-              {% endif %}
-            </div>
-          {% endif %}
         </div>
+        {% if bookmark_list.show_preview_images and bookmark_item.preview_image_file %}
+          <img class="preview-image" src="{% static bookmark_item.preview_image_file %}"/>
+        {% endif %}
       </li>
     {% endfor %}
   </ul>

bookmarks/templates/bookmarks/details/form.html

@@ -37,6 +37,11 @@
       </a>
     {% endif %}
   </div>
+  {% if details.preview_image_enabled and details.bookmark.preview_image_file %}
+    <div class="preview-image">
+      <img src="{% static details.bookmark.preview_image_file %}"/>
+    </div>
+  {% endif %}
   <dl class="grid columns-2 columns-sm-1 gap-0">
     {% if details.is_editable %}
       <div class="status col-2">

bookmarks/templates/settings/general.html

@@ -127,6 +127,15 @@
             <button class="btn mt-2" name="refresh_favicons">Refresh Favicons</button>
           {% endif %}
         </div>
+        <div class="form-group">
+          <label for="{{ form.enable_preview_images.id_for_label }}" class="form-checkbox">
+            {{ form.enable_preview_images }}
+            <i class="form-icon"></i> Enable Preview Images
+          </label>
+          <div class="form-input-hint">
+            Automatically loads preview images for bookmarked websites and displays them next to each bookmark.
+          </div>
+        </div>
         <div class="form-group">
           <label for="{{ form.web_archive_integration.id_for_label }}" class="form-label">Internet Archive
             integration</label>

bookmarks/tests/helpers.py

@@ -39,6 +39,7 @@ class BookmarkFactoryMixin:
         website_description: str = "",
         web_archive_snapshot_url: str = "",
         favicon_file: str = "",
+        preview_image_file: str = "",
         added: datetime = None,
     ):
         if title is None:
@@ -67,6 +68,7 @@ class BookmarkFactoryMixin:
             shared=shared,
             web_archive_snapshot_url=web_archive_snapshot_url,
             favicon_file=favicon_file,
+            preview_image_file=preview_image_file,
         )
         bookmark.save()
         for tag in tags:

bookmarks/tests/test_bookmark_details_modal.py

@@ -300,6 +300,36 @@ class BookmarkDetailsModalTestCase(TestCase, BookmarkFactoryMixin, HtmlTestMixin
             web_archive_link["target"], UserProfile.BOOKMARK_LINK_TARGET_SELF
         )
 
+    def test_preview_image(self):
+        # without image
+        bookmark = self.setup_bookmark()
+        soup = self.get_details(bookmark)
+        image = soup.select_one("div.preview-image img")
+        self.assertIsNone(image)
+
+        # with image
+        bookmark = self.setup_bookmark(preview_image_file="example.png")
+        soup = self.get_details(bookmark)
+        image = soup.select_one("div.preview-image img")
+        self.assertIsNone(image)
+
+        # preview images enabled, no image
+        profile = self.get_or_create_test_user().profile
+        profile.enable_preview_images = True
+        profile.save()
+
+        bookmark = self.setup_bookmark()
+        soup = self.get_details(bookmark)
+        image = soup.select_one("div.preview-image img")
+        self.assertIsNone(image)
+
+        # preview images enabled, image present
+        bookmark = self.setup_bookmark(preview_image_file="example.png")
+        soup = self.get_details(bookmark)
+        image = soup.select_one("div.preview-image img")
+        self.assertIsNotNone(image)
+        self.assertEqual(image["src"], "/static/example.png")
+
     def test_status(self):
         # renders form
         bookmark = self.setup_bookmark()

bookmarks/tests/test_bookmarks_api.py

@@ -628,7 +628,10 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin):
             website_loader, "load_website_metadata"
         ) as mock_load_website_metadata:
             expected_metadata = WebsiteMetadata(
-                "https://example.com", "Scraped metadata", "Scraped description"
+                "https://example.com",
+                "Scraped metadata",
+                "Scraped description",
+                "https://example.com/preview.png",
             )
             mock_load_website_metadata.return_value = expected_metadata
 
@@ -640,9 +643,10 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin):
             metadata = response.data["metadata"]
 
             self.assertIsNotNone(metadata)
-            self.assertIsNotNone(expected_metadata.url, metadata["url"])
+            self.assertEqual(expected_metadata.url, metadata["url"])
-            self.assertIsNotNone(expected_metadata.title, metadata["title"])
+            self.assertEqual(expected_metadata.title, metadata["title"])
-            self.assertIsNotNone(expected_metadata.description, metadata["description"])
+            self.assertEqual(expected_metadata.description, metadata["description"])
+            self.assertEqual(expected_metadata.preview_image, metadata["preview_image"])
 
     def test_check_returns_bookmark_if_url_is_bookmarked(self):
         self.authenticate()
@@ -687,9 +691,10 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin):
 
             mock_load_website_metadata.assert_not_called()
             self.assertIsNotNone(metadata)
-            self.assertIsNotNone(bookmark.url, metadata["url"])
+            self.assertEqual(bookmark.url, metadata["url"])
-            self.assertIsNotNone(bookmark.website_title, metadata["title"])
+            self.assertEqual(bookmark.website_title, metadata["title"])
-            self.assertIsNotNone(bookmark.website_description, metadata["description"])
+            self.assertEqual(bookmark.website_description, metadata["description"])
+            self.assertIsNone(metadata["preview_image"])
 
     def test_can_only_access_own_bookmarks(self):
         self.authenticate()

bookmarks/tests/test_bookmarks_list_template.py

@@ -20,7 +20,7 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
         self, html: str, bookmark: Bookmark, link_target: str = "_blank"
     ):
         favicon_img = (
-            f'<img src="/static/{bookmark.favicon_file}" alt="">'
+            f'<img class="favicon" src="/static/{bookmark.favicon_file}" alt="">'
             if bookmark.favicon_file
             else ""
         )
@@ -148,19 +148,41 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
         )
 
     def assertFaviconVisible(self, html: str, bookmark: Bookmark):
-        self.assertFaviconCount(html, bookmark, 1)
+        self.assertFavicon(html, bookmark, True)
 
     def assertFaviconHidden(self, html: str, bookmark: Bookmark):
-        self.assertFaviconCount(html, bookmark, 0)
+        self.assertFavicon(html, bookmark, False)
 
-    def assertFaviconCount(self, html: str, bookmark: Bookmark, count=1):
+    def assertFavicon(self, html: str, bookmark: Bookmark, visible=True):
-        self.assertInHTML(
+        soup = self.make_soup(html)
-            f"""
+
-            <img src="/static/{bookmark.favicon_file}" alt="">
+        favicon = soup.select_one(".favicon")
-            """,
+
-            html,
+        if not visible:
-            count=count,
+            self.assertIsNone(favicon)
-        )
+            return
+
+        url = f"/static/{bookmark.favicon_file}"
+        self.assertIsNotNone(favicon)
+        self.assertEqual(favicon["src"], url)
+
+    def assertPreviewImageVisible(self, html: str, bookmark: Bookmark):
+        self.assertPreviewImage(html, bookmark, True)
+
+    def assertPreviewImageHidden(self, html: str, bookmark: Bookmark):
+        self.assertPreviewImage(html, bookmark, False)
+
+    def assertPreviewImage(self, html: str, bookmark: Bookmark, visible=True):
+        soup = self.make_soup(html)
+        preview_image = soup.select_one(".preview-image")
+
+        if not visible:
+            self.assertIsNone(preview_image)
+            return
+
+        url = f"/static/{bookmark.preview_image_file}"
+        self.assertIsNotNone(preview_image)
+        self.assertEqual(preview_image["src"], url)
 
     def assertBookmarkURLCount(
         self, html: str, bookmark: Bookmark, link_target: str = "_blank", count=0
@@ -640,6 +662,36 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
             html,
         )
 
+    def test_preview_image_should_be_visible_when_preview_images_enabled(self):
+        profile = self.get_or_create_test_user().profile
+        profile.enable_preview_images = True
+        profile.save()
+
+        bookmark = self.setup_bookmark(preview_image_file="preview.png")
+        html = self.render_template()
+
+        self.assertPreviewImageVisible(html, bookmark)
+
+    def test_preview_image_should_be_hidden_when_preview_images_disabled(self):
+        profile = self.get_or_create_test_user().profile
+        profile.enable_preview_images = False
+        profile.save()
+
+        bookmark = self.setup_bookmark(preview_image_file="preview.png")
+        html = self.render_template()
+
+        self.assertPreviewImageHidden(html, bookmark)
+
+    def test_preview_image_should_be_hidden_when_there_is_no_preview_image(self):
+        profile = self.get_or_create_test_user().profile
+        profile.enable_preview_images = True
+        profile.save()
+
+        bookmark = self.setup_bookmark()
+        html = self.render_template()
+
+        self.assertPreviewImageHidden(html, bookmark)
+
     def test_favicon_should_be_visible_when_favicons_enabled(self):
         profile = self.get_or_create_test_user().profile
         profile.enable_favicons = True

bookmarks/tests/test_bookmarks_service.py

@@ -42,7 +42,10 @@ class BookmarkServiceTestCase(TestCase, BookmarkFactoryMixin):
             website_loader, "load_website_metadata"
         ) as mock_load_website_metadata:
             expected_metadata = WebsiteMetadata(
-                "https://example.com", "Website title", "Website description"
+                "https://example.com",
+                "Website title",
+                "Website description",
+                "https://example.com/preview.png",
             )
             mock_load_website_metadata.return_value = expected_metadata
 
@@ -157,6 +160,7 @@ class BookmarkServiceTestCase(TestCase, BookmarkFactoryMixin):
                 "https://example.com/updated",
                 "Updated website title",
                 "Updated website description",
+                "https://example.com/preview.png",
             )
             mock_load_website_metadata.return_value = expected_metadata
 

bookmarks/tests/test_bookmarks_tasks.py

@@ -74,11 +74,18 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
             self.mock_singlefile_create_snapshot_patcher.start()
         )
 
+        self.mock_load_preview_image_patcher = mock.patch(
+            "bookmarks.services.preview_image_loader.load_preview_image"
+        )
+        self.mock_load_preview_image = self.mock_load_preview_image_patcher.start()
+        self.mock_load_preview_image.return_value = "preview_image.png"
+
         user = self.get_or_create_test_user()
         user.profile.web_archive_integration = (
             UserProfile.WEB_ARCHIVE_INTEGRATION_ENABLED
         )
         user.profile.enable_favicons = True
+        user.profile.enable_preview_images = True
         user.profile.save()
 
     def tearDown(self):
@@ -86,6 +93,7 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
         self.mock_cdx_api_patcher.stop()
         self.mock_load_favicon_patcher.stop()
         self.mock_singlefile_create_snapshot_patcher.stop()
+        self.mock_load_preview_image_patcher.stop()
         huey.storage.flush_results()
         huey.immediate = False
 
@@ -507,6 +515,69 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
 
         self.assertEqual(self.executed_count(), 0)
 
+    def test_load_preview_image_should_create_preview_image_file(self):
+        bookmark = self.setup_bookmark()
+
+        tasks.load_preview_image(self.get_or_create_test_user(), bookmark)
+        bookmark.refresh_from_db()
+
+        self.assertEqual(self.executed_count(), 1)
+        self.assertEqual(bookmark.preview_image_file, "preview_image.png")
+
+    def test_load_preview_image_should_update_preview_image_file(self):
+        bookmark = self.setup_bookmark(
+            preview_image_file="preview_image.png",
+        )
+
+        self.mock_load_preview_image.return_value = "preview_image_upd.png"
+
+        tasks.load_preview_image(self.get_or_create_test_user(), bookmark)
+
+        bookmark.refresh_from_db()
+        self.mock_load_preview_image.assert_called_once()
+        self.assertEqual(bookmark.preview_image_file, "preview_image_upd.png")
+
+    def test_load_preview_image_should_handle_missing_bookmark(self):
+        tasks._load_preview_image_task(123)
+
+        self.mock_load_preview_image.assert_not_called()
+
+    def test_load_preview_image_should_not_save_stale_bookmark_data(self):
+        bookmark = self.setup_bookmark()
+
+        # update bookmark during API call to check that saving
+        # the image does not overwrite updated bookmark data
+        def mock_load_preview_image_impl(url):
+            bookmark.title = "Updated title"
+            bookmark.save()
+            return "test.png"
+
+        self.mock_load_preview_image.side_effect = mock_load_preview_image_impl
+
+        tasks.load_preview_image(self.get_or_create_test_user(), bookmark)
+        bookmark.refresh_from_db()
+
+        self.assertEqual(bookmark.title, "Updated title")
+        self.assertEqual(bookmark.preview_image_file, "test.png")
+
+    @override_settings(LD_DISABLE_BACKGROUND_TASKS=True)
+    def test_load_preview_image_should_not_run_when_background_tasks_are_disabled(self):
+        bookmark = self.setup_bookmark()
+        tasks.load_preview_image(self.get_or_create_test_user(), bookmark)
+
+        self.assertEqual(self.executed_count(), 0)
+
+    def test_load_preview_image_should_not_run_when_preview_image_feature_is_disabled(
+        self,
+    ):
+        self.user.profile.enable_preview_images = False
+        self.user.profile.save()
+
+        bookmark = self.setup_bookmark()
+        tasks.load_preview_image(self.get_or_create_test_user(), bookmark)
+
+        self.assertEqual(self.executed_count(), 0)
+
     @override_settings(LD_ENABLE_SNAPSHOTS=True)
     def test_create_html_snapshot_should_create_pending_asset(self):
         bookmark = self.setup_bookmark()

bookmarks/tests/test_preview_image_loader.py

@@ -0,0 +1,108 @@
+import io
+import os
+import tempfile
+from pathlib import Path
+from unittest import mock
+
+from django.conf import settings
+from django.test import TestCase
+
+from bookmarks.services import preview_image_loader
+
+mock_image_data = b"mock_image"
+
+
+class MockStreamingResponse:
+    def __init__(self, data=mock_image_data, content_type="image/png"):
+        self.chunks = [data]
+        self.headers = {"Content-Type": content_type}
+
+    def iter_content(self, **kwargs):
+        return self.chunks
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        pass
+
+
+class PreviewImageLoaderTestCase(TestCase):
+    def setUp(self) -> None:
+        self.temp_folder = tempfile.TemporaryDirectory()
+        self.settings_override = self.settings(LD_PREVIEW_FOLDER=self.temp_folder.name)
+        self.settings_override.enable()
+        self.mock_load_website_metadata_patcher = mock.patch(
+            "bookmarks.services.website_loader.load_website_metadata"
+        )
+        self.mock_load_website_metadata = (
+            self.mock_load_website_metadata_patcher.start()
+        )
+        self.mock_load_website_metadata.return_value = mock.Mock(
+            preview_image="https://example.com/image.png"
+        )
+
+    def tearDown(self) -> None:
+        self.temp_folder.cleanup()
+        self.settings_override.disable()
+        self.mock_load_website_metadata_patcher.stop()
+
+    def create_mock_response(self, icon_data=mock_image_data, content_type="image/png"):
+        mock_response = mock.Mock()
+        mock_response.raw = io.BytesIO(icon_data)
+        return MockStreamingResponse(icon_data, content_type)
+
+    def get_image_path(self, filename):
+        return Path(os.path.join(settings.LD_PREVIEW_FOLDER, filename))
+
+    def image_exists(self, filename):
+        return self.get_image_path(filename).exists()
+
+    def get_image_data(self, filename):
+        return self.get_image_path(filename).read_bytes()
+
+    def test_load_preview_image(self):
+        with mock.patch("requests.get") as mock_get:
+            mock_get.return_value = self.create_mock_response()
+
+            file = preview_image_loader.load_preview_image("https://example.com")
+
+            self.assertTrue(self.image_exists(file))
+            self.assertEqual(mock_image_data, self.get_image_data(file))
+
+    def test_load_preview_image_returns_none_if_no_preview_image_detected(self):
+        with mock.patch("requests.get") as mock_get:
+            mock_get.return_value = self.create_mock_response()
+            self.mock_load_website_metadata.return_value = mock.Mock(preview_image=None)
+
+            file = preview_image_loader.load_preview_image("https://example.com")
+
+            self.assertIsNone(file)
+
+    def test_load_preview_image_creates_folder_if_not_exists(self):
+        with mock.patch("requests.get") as mock_get:
+            mock_get.return_value = self.create_mock_response()
+
+            folder = Path(settings.LD_PREVIEW_FOLDER)
+            folder.rmdir()
+
+            self.assertFalse(folder.exists())
+
+            preview_image_loader.load_preview_image("https://example.com")
+
+            self.assertTrue(folder.exists())
+
+    def test_guess_file_extension(self):
+        with mock.patch("requests.get") as mock_get:
+            mock_get.return_value = self.create_mock_response(content_type="image/png")
+            file = preview_image_loader.load_preview_image("https://example.com")
+
+            self.assertTrue(self.image_exists(file))
+            self.assertEqual("png", file.split(".")[-1])
+
+        with mock.patch("requests.get") as mock_get:
+            mock_get.return_value = self.create_mock_response(content_type="image/jpeg")
+            file = preview_image_loader.load_preview_image("https://example.com")
+
+            self.assertTrue(self.image_exists(file))
+            self.assertEqual("jpg", file.split(".")[-1])

bookmarks/tests/test_website_loader.py

@@ -29,7 +29,9 @@ class WebsiteLoaderTestCase(TestCase):
         # clear cached metadata before test run
         website_loader.load_website_metadata.cache_clear()
 
-    def render_html_document(self, title, description="", og_description=""):
+    def render_html_document(
+        self, title, description="", og_description="", og_image=""
+    ):
         meta_description = (
             f'<meta name="description" content="{description}">' if description else ""
         )
@@ -38,6 +40,9 @@ class WebsiteLoaderTestCase(TestCase):
             if og_description
             else ""
         )
+        meta_og_image = (
+            f'<meta property="og:image" content="{og_image}">' if og_image else ""
+        )
         return f"""
         <!DOCTYPE html>
         <html lang="en">
@@ -46,6 +51,7 @@ class WebsiteLoaderTestCase(TestCase):
             <title>{title}</title>
             {meta_description}
             {meta_og_description}
+            {meta_og_image}
         </head>
         <body></body>
         </html>
@@ -105,6 +111,7 @@ class WebsiteLoaderTestCase(TestCase):
             metadata = website_loader.load_website_metadata("https://example.com")
             self.assertEqual("test title", metadata.title)
             self.assertEqual("test description", metadata.description)
+            self.assertIsNone(metadata.preview_image)
 
     def test_load_website_metadata_trims_title_and_description(self):
         with mock.patch(
@@ -128,6 +135,44 @@ class WebsiteLoaderTestCase(TestCase):
             self.assertEqual("test title", metadata.title)
             self.assertEqual("test og description", metadata.description)
 
+    def test_load_website_metadata_using_og_image(self):
+        with mock.patch(
+            "bookmarks.services.website_loader.load_page"
+        ) as mock_load_page:
+            mock_load_page.return_value = self.render_html_document(
+                "test title", og_image="http://example.com/image.jpg"
+            )
+            metadata = website_loader.load_website_metadata("https://example.com")
+            self.assertEqual("http://example.com/image.jpg", metadata.preview_image)
+
+    def test_load_website_metadata_gets_absolute_og_image_path_when_path_starts_with_dots(
+        self,
+    ):
+        with mock.patch(
+            "bookmarks.services.website_loader.load_page"
+        ) as mock_load_page:
+            mock_load_page.return_value = self.render_html_document(
+                "test title", og_image="../image.jpg"
+            )
+            metadata = website_loader.load_website_metadata(
+                "https://example.com/a/b/page.html"
+            )
+            self.assertEqual("https://example.com/a/image.jpg", metadata.preview_image)
+
+    def test_load_website_metadata_gets_absolute_og_image_path_when_path_starts_with_slash(
+        self,
+    ):
+        with mock.patch(
+            "bookmarks.services.website_loader.load_page"
+        ) as mock_load_page:
+            mock_load_page.return_value = self.render_html_document(
+                "test title", og_image="/image.jpg"
+            )
+            metadata = website_loader.load_website_metadata(
+                "https://example.com/a/b/page.html"
+            )
+            self.assertEqual("https://example.com/image.jpg", metadata.preview_image)
+
     def test_load_website_metadata_prefers_description_over_og_description(self):
         with mock.patch(
             "bookmarks.services.website_loader.load_page"

bookmarks/views/partials/contexts.py

@@ -145,6 +145,7 @@ class BookmarkItem:
         self.tag_names = bookmark.tag_names
         self.web_archive_snapshot_url = bookmark.web_archive_snapshot_url
         self.favicon_file = bookmark.favicon_file
+        self.preview_image_file = bookmark.preview_image_file
         self.is_archived = bookmark.is_archived
         self.unread = bookmark.unread
         self.owner = bookmark.owner
@@ -215,6 +216,7 @@ class BookmarkListContext:
         self.show_archive_action = user_profile.display_archive_bookmark_action
         self.show_remove_action = user_profile.display_remove_bookmark_action
         self.show_favicons = user_profile.enable_favicons
+        self.show_preview_images = user_profile.enable_preview_images
         self.show_notes = user_profile.permanent_notes
 
     @staticmethod
@@ -384,6 +386,7 @@ class BookmarkDetailsContext:
         self.profile = request.user_profile
         self.is_editable = bookmark.owner == user
         self.sharing_enabled = user_profile.enable_sharing
+        self.preview_image_enabled = user_profile.enable_preview_images
         self.show_link_icons = user_profile.enable_favicons and bookmark.favicon_file
         # For now hide files section if snapshots are not supported
         self.show_files = settings.LD_ENABLE_SNAPSHOTS

bootstrap.sh

@@ -7,6 +7,8 @@ LD_SERVER_PORT="${LD_SERVER_PORT:-9090}"
 mkdir -p data
 # Create favicon folder if it does not exist
 mkdir -p data/favicons
+# Create previews folder if it does not exist
+mkdir -p data/previews
 # Create assets folder if it does not exist
 mkdir -p data/assets
 

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "linkding",
-  "version": "1.25.0",
+  "version": "1.30.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "linkding",
-      "version": "1.25.0",
+      "version": "1.30.0",
       "license": "MIT",
       "dependencies": {
         "@rollup/plugin-node-resolve": "^15.2.3",

siteroot/settings/base.py

@@ -144,6 +144,7 @@ STATICFILES_FINDERS = [
 STATICFILES_DIRS = [
     os.path.join(BASE_DIR, "bookmarks", "styles"),
     os.path.join(BASE_DIR, "data", "favicons"),
+    os.path.join(BASE_DIR, "data", "previews"),
 ]
 
 # REST framework
@@ -286,6 +287,9 @@ LD_ENABLE_REFRESH_FAVICONS = os.getenv("LD_ENABLE_REFRESH_FAVICONS", True) in (
     "1",
 )
 
+# Previews settings
+LD_PREVIEW_FOLDER = os.path.join(BASE_DIR, "data", "previews")
+
 # Asset / snapshot settings
 LD_ASSET_FOLDER = os.path.join(BASE_DIR, "data", "assets")
 

uwsgi.ini

@@ -3,6 +3,7 @@ module = siteroot.wsgi:application
 env = DJANGO_SETTINGS_MODULE=siteroot.settings.prod
 static-map = /static=static
 static-map = /static=data/favicons
+static-map = /static=data/previews
 processes = 2
 threads = 2
 pidfile = /tmp/linkding.pid
@@ -16,6 +17,7 @@ die-on-term = true
 if-env = LD_CONTEXT_PATH
 static-map = /%(_)static=static
 static-map = /%(_)static=data/favicons
+static-map = /%(_)static=data/previews
 endif =
 
 if-env = LD_REQUEST_TIMEOUT