diff --git a/.gitignore b/.gitignore index f7dd1e1..a9663a7 100644 --- a/.gitignore +++ b/.gitignore @@ -194,3 +194,5 @@ typings/ # ublock + chromium /uBlock0.chromium /chromium-profile +# direnv +/.direnv diff --git a/bookmarks/api/routes.py b/bookmarks/api/routes.py index bda6471..41f7321 100644 --- a/bookmarks/api/routes.py +++ b/bookmarks/api/routes.py @@ -99,7 +99,10 @@ class BookmarkViewSet( # Either return metadata from existing bookmark, or scrape from URL if bookmark: metadata = WebsiteMetadata( - url, bookmark.website_title, bookmark.website_description + url, + bookmark.website_title, + bookmark.website_description, + None, ) else: metadata = website_loader.load_website_metadata(url) diff --git a/bookmarks/migrations/0034_bookmark_preview_image_file_and_more.py b/bookmarks/migrations/0034_bookmark_preview_image_file_and_more.py new file mode 100644 index 0000000..1e3aca7 --- /dev/null +++ b/bookmarks/migrations/0034_bookmark_preview_image_file_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0.3 on 2024-05-07 07:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookmarks", "0033_userprofile_default_mark_unread"), + ] + + operations = [ + migrations.AddField( + model_name="bookmark", + name="preview_image_file", + field=models.CharField(blank=True, max_length=512, null=True), + ), + migrations.AddField( + model_name="userprofile", + name="enable_preview_images", + field=models.BooleanField(default=False), + ), + ] diff --git a/bookmarks/models.py b/bookmarks/models.py index b922bd2..b15963c 100644 --- a/bookmarks/models.py +++ b/bookmarks/models.py @@ -59,6 +59,7 @@ class Bookmark(models.Model): website_description = models.TextField(blank=True, null=True) web_archive_snapshot_url = models.CharField(max_length=2048, blank=True) favicon_file = models.CharField(max_length=512, blank=True) + preview_image_file = models.CharField(max_length=512, blank=True, null=True) unread = models.BooleanField(default=False) is_archived = models.BooleanField(default=False) shared = models.BooleanField(default=False) @@ -394,6 +395,7 @@ class UserProfile(models.Model): enable_sharing = models.BooleanField(default=False, null=False) enable_public_sharing = models.BooleanField(default=False, null=False) enable_favicons = models.BooleanField(default=False, null=False) + enable_preview_images = models.BooleanField(default=False, null=False) display_url = models.BooleanField(default=False, null=False) display_view_bookmark_action = models.BooleanField(default=True, null=False) display_edit_bookmark_action = models.BooleanField(default=True, null=False) @@ -420,6 +422,7 @@ class UserProfileForm(forms.ModelForm): "enable_sharing", "enable_public_sharing", "enable_favicons", + "enable_preview_images", "enable_automatic_html_snapshots", "display_url", "display_view_bookmark_action", diff --git a/bookmarks/services/bookmarks.py b/bookmarks/services/bookmarks.py index 72ec16d..5e96dbb 100644 --- a/bookmarks/services/bookmarks.py +++ b/bookmarks/services/bookmarks.py @@ -40,6 +40,8 @@ def create_bookmark(bookmark: Bookmark, tag_string: str, current_user: User): tasks.create_web_archive_snapshot(current_user, bookmark, False) # Load favicon tasks.load_favicon(current_user, bookmark) + # Load preview image + tasks.load_preview_image(current_user, bookmark) # Create HTML snapshot if current_user.profile.enable_automatic_html_snapshots: tasks.create_html_snapshot(bookmark) @@ -58,6 +60,8 @@ def update_bookmark(bookmark: Bookmark, tag_string, current_user: User): bookmark.save() # Update favicon tasks.load_favicon(current_user, bookmark) + # Update preview image + tasks.load_preview_image(current_user, bookmark) if has_url_changed: # Update web archive snapshot, if URL changed diff --git a/bookmarks/services/preview_image_loader.py b/bookmarks/services/preview_image_loader.py new file mode 100644 index 0000000..2311f65 --- /dev/null +++ b/bookmarks/services/preview_image_loader.py @@ -0,0 +1,46 @@ +import logging +import mimetypes +import os.path +import hashlib +from pathlib import Path + +import requests +from django.conf import settings +from bookmarks.services import website_loader + +logger = logging.getLogger(__name__) + + +def _ensure_preview_folder(): + Path(settings.LD_PREVIEW_FOLDER).mkdir(parents=True, exist_ok=True) + + +def _url_to_filename(preview_image: str) -> str: + return hashlib.md5(preview_image.encode()).hexdigest() + + +def _get_image_path(preview_image_file: str) -> Path: + return Path(os.path.join(settings.LD_PREVIEW_FOLDER, preview_image_file)) + + +def load_preview_image(url: str) -> str | None: + _ensure_preview_folder() + + metadata = website_loader.load_website_metadata(url) + if not metadata.preview_image: + logger.debug(f"Could not find preview image in metadata: {url}") + return None + + logger.debug(f"Loading preview image: {metadata.preview_image}") + with requests.get(metadata.preview_image, stream=True) as response: + content_type = response.headers["Content-Type"] + preview_image_hash = _url_to_filename(url) + file_extension = mimetypes.guess_extension(content_type) + preview_image_file = f"{preview_image_hash}{file_extension}" + preview_image_path = _get_image_path(preview_image_file) + with open(preview_image_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + file.write(chunk) + logger.debug(f"Saved preview image as: {preview_image_path}") + + return preview_image_file diff --git a/bookmarks/services/tasks.py b/bookmarks/services/tasks.py index e97f55c..e42c942 100644 --- a/bookmarks/services/tasks.py +++ b/bookmarks/services/tasks.py @@ -15,7 +15,7 @@ from waybackpy.exceptions import WaybackError, TooManyRequestsError, NoCDXRecord import bookmarks.services.wayback from bookmarks.models import Bookmark, BookmarkAsset, UserProfile -from bookmarks.services import favicon_loader, singlefile +from bookmarks.services import favicon_loader, singlefile, preview_image_loader from bookmarks.services.website_loader import DEFAULT_USER_AGENT logger = logging.getLogger(__name__) @@ -221,6 +221,30 @@ def _schedule_refresh_favicons_task(user_id: int): _load_favicon_task(bookmark.id) +def load_preview_image(user: User, bookmark: Bookmark): + if user.profile.enable_preview_images and not settings.LD_DISABLE_BACKGROUND_TASKS: + _load_preview_image_task(bookmark.id) + + +@task() +def _load_preview_image_task(bookmark_id: int): + try: + bookmark = Bookmark.objects.get(id=bookmark_id) + except Bookmark.DoesNotExist: + return + + logger.info(f"Load preview image for bookmark. url={bookmark.url}") + + new_preview_image_file = preview_image_loader.load_preview_image(bookmark.url) + + if new_preview_image_file != bookmark.preview_image_file: + bookmark.preview_image_file = new_preview_image_file + bookmark.save(update_fields=["preview_image_file"]) + logger.info( + f"Successfully updated preview image for bookmark. url={bookmark.url} preview_image_file={new_preview_image_file}" + ) + + def is_html_snapshot_feature_active() -> bool: return settings.LD_ENABLE_SNAPSHOTS and not settings.LD_DISABLE_BACKGROUND_TASKS diff --git a/bookmarks/services/website_loader.py b/bookmarks/services/website_loader.py index 644e2ea..389e374 100644 --- a/bookmarks/services/website_loader.py +++ b/bookmarks/services/website_loader.py @@ -1,6 +1,7 @@ import logging from dataclasses import dataclass from functools import lru_cache +from urllib.parse import urljoin import requests from bs4 import BeautifulSoup @@ -15,12 +16,14 @@ class WebsiteMetadata: url: str title: str description: str + preview_image: str | None def to_dict(self): return { "url": self.url, "title": self.title, "description": self.description, + "preview_image": self.preview_image, } @@ -30,6 +33,7 @@ class WebsiteMetadata: def load_website_metadata(url: str): title = None description = None + preview_image = None try: start = timezone.now() page_text = load_page(url) @@ -55,10 +59,21 @@ def load_website_metadata(url: str): else None ) + image_tag = soup.find("meta", attrs={"property": "og:image"}) + preview_image = image_tag["content"].strip() if image_tag else None + if ( + preview_image + and not preview_image.startswith("http://") + and not preview_image.startswith("https://") + ): + preview_image = urljoin(url, preview_image) + end = timezone.now() logger.debug(f"Parsing duration: {end - start}") finally: - return WebsiteMetadata(url=url, title=title, description=description) + return WebsiteMetadata( + url=url, title=title, description=description, preview_image=preview_image + ) CHUNK_SIZE = 50 * 1024 diff --git a/bookmarks/styles/bookmark-details.scss b/bookmarks/styles/bookmark-details.scss index d19ed1a..25aee2a 100644 --- a/bookmarks/styles/bookmark-details.scss +++ b/bookmarks/styles/bookmark-details.scss @@ -33,6 +33,15 @@ text-overflow: ellipsis; } + .preview-image { + margin: $unit-4 0; + + img { + max-width: 100%; + max-height: 200px; + } + } + dl { margin-bottom: 0; } diff --git a/bookmarks/styles/bookmark-page.scss b/bookmarks/styles/bookmark-page.scss index 8ce1e03..32797d2 100644 --- a/bookmarks/styles/bookmark-page.scss +++ b/bookmarks/styles/bookmark-page.scss @@ -128,8 +128,25 @@ ul.bookmark-list { /* Bookmarks */ li[ld-bookmark-item] { position: relative; + display: flex; + gap: $unit-2; margin-top: $unit-2; + .content { + flex: 1 1 0; + min-width: 0; + } + + img.preview-image { + flex: 0 0 auto; + width: 100px; + height: 60px; + margin-top: $unit-h; + object-fit: cover; + border-radius: $border-radius; + border: solid 1px $border-color-dark; + } + .form-checkbox.bulk-edit-checkbox { display: none; } @@ -346,7 +363,7 @@ $bulk-edit-transition-duration: 400ms; transition: all $bulk-edit-transition-duration; .form-icon { - top: 0; + top: 0; } } diff --git a/bookmarks/templates/bookmarks/bookmark_list.html b/bookmarks/templates/bookmarks/bookmark_list.html index 6e3214e..dadfae0 100644 --- a/bookmarks/templates/bookmarks/bookmark_list.html +++ b/bookmarks/templates/bookmarks/bookmark_list.html @@ -10,138 +10,143 @@ data-bookmarks-total="{{ bookmark_list.bookmarks_total }}"> {% for bookmark_item in bookmark_list.items %}
  • -
    - - {% if bookmark_item.favicon_file and bookmark_list.show_favicons %} - - {% endif %} - - {{ bookmark_item.title }} - -
    - {% if bookmark_list.show_url %} -
    - - {{ bookmark_item.url }} +
    +
    + + {% if bookmark_item.favicon_file and bookmark_list.show_favicons %} + + {% endif %} + + {{ bookmark_item.title }}
    - {% endif %} - {% if bookmark_list.description_display == 'inline' %} -
    + {% if bookmark_list.show_url %} +
    + + {{ bookmark_item.url }} + +
    + {% endif %} + {% if bookmark_list.description_display == 'inline' %} +
    + {% if bookmark_item.tag_names %} + + {% for tag_name in bookmark_item.tag_names %} + {{ tag_name|hash_tag }} + {% endfor %} + + {% endif %} + {% if bookmark_item.tag_names and bookmark_item.description %} | {% endif %} + {% if bookmark_item.description %} + {{ bookmark_item.description }} + {% endif %} +
    + {% else %} + {% if bookmark_item.description %} +
    {{ bookmark_item.description }}
    + {% endif %} {% if bookmark_item.tag_names %} - +
    {% for tag_name in bookmark_item.tag_names %} {{ tag_name|hash_tag }} {% endfor %} +
    + {% endif %} + {% endif %} + {% if bookmark_item.notes %} +
    +
    {% markdown bookmark_item.notes %}
    +
    + {% endif %} +
    + {% if bookmark_item.display_date %} + {% if bookmark_item.web_archive_snapshot_url %} + + {{ bookmark_item.display_date }} ∞ + + {% else %} + {{ bookmark_item.display_date }} + {% endif %} + | + {% endif %} + {# View link is visible for both owned and shared bookmarks #} + {% if bookmark_list.show_view_action %} + View + {% endif %} + {% if bookmark_item.is_editable %} + {# Bookmark owner actions #} + {% if bookmark_list.show_edit_action %} + Edit + {% endif %} + {% if bookmark_list.show_archive_action %} + {% if bookmark_item.is_archived %} + + {% else %} + + {% endif %} + {% endif %} + {% if bookmark_list.show_remove_action %} + + {% endif %} + {% else %} + {# Shared bookmark actions #} + Shared by + {{ bookmark_item.owner.username }} {% endif %} - {% if bookmark_item.tag_names and bookmark_item.description %} | {% endif %} - {% if bookmark_item.description %} - {{ bookmark_item.description }} + {% if bookmark_item.has_extra_actions %} +
    + | + {% if bookmark_item.show_mark_as_read %} + + {% endif %} + {% if bookmark_item.show_unshare %} + + {% endif %} + {% if bookmark_item.show_notes_button %} + + {% endif %} +
    {% endif %}
    - {% else %} - {% if bookmark_item.description %} -
    {{ bookmark_item.description }}
    - {% endif %} - {% if bookmark_item.tag_names %} -
    - {% for tag_name in bookmark_item.tag_names %} - {{ tag_name|hash_tag }} - {% endfor %} -
    - {% endif %} - {% endif %} - {% if bookmark_item.notes %} -
    -
    {% markdown bookmark_item.notes %}
    -
    - {% endif %} -
    - {% if bookmark_item.display_date %} - {% if bookmark_item.web_archive_snapshot_url %} - - {{ bookmark_item.display_date }} ∞ - - {% else %} - {{ bookmark_item.display_date }} - {% endif %} - | - {% endif %} - {# View link is visible for both owned and shared bookmarks #} - {% if bookmark_list.show_view_action %} - View - {% endif %} - {% if bookmark_item.is_editable %} - {# Bookmark owner actions #} - {% if bookmark_list.show_edit_action %} - Edit - {% endif %} - {% if bookmark_list.show_archive_action %} - {% if bookmark_item.is_archived %} - - {% else %} - - {% endif %} - {% endif %} - {% if bookmark_list.show_remove_action %} - - {% endif %} - {% else %} - {# Shared bookmark actions #} - Shared by - {{ bookmark_item.owner.username }} - - {% endif %} - {% if bookmark_item.has_extra_actions %} -
    - | - {% if bookmark_item.show_mark_as_read %} - - {% endif %} - {% if bookmark_item.show_unshare %} - - {% endif %} - {% if bookmark_item.show_notes_button %} - - {% endif %} -
    - {% endif %}
    + {% if bookmark_list.show_preview_images and bookmark_item.preview_image_file %} + + {% endif %}
    • {% endfor %} diff --git a/bookmarks/templates/bookmarks/details/form.html b/bookmarks/templates/bookmarks/details/form.html index b61f6c5..e00fcf4 100644 --- a/bookmarks/templates/bookmarks/details/form.html +++ b/bookmarks/templates/bookmarks/details/form.html @@ -37,6 +37,11 @@ {% endif %} + {% if details.preview_image_enabled and details.bookmark.preview_image_file %} +
    + +
    + {% endif %}
    {% if details.is_editable %}
    diff --git a/bookmarks/templates/settings/general.html b/bookmarks/templates/settings/general.html index 436f404..d021c31 100644 --- a/bookmarks/templates/settings/general.html +++ b/bookmarks/templates/settings/general.html @@ -127,6 +127,15 @@ {% endif %}
    +
    + +
    + Automatically loads preview images for bookmarked websites and displays them next to each bookmark. +
    +
    diff --git a/bookmarks/tests/helpers.py b/bookmarks/tests/helpers.py index d881fb9..4cd160d 100644 --- a/bookmarks/tests/helpers.py +++ b/bookmarks/tests/helpers.py @@ -39,6 +39,7 @@ class BookmarkFactoryMixin: website_description: str = "", web_archive_snapshot_url: str = "", favicon_file: str = "", + preview_image_file: str = "", added: datetime = None, ): if title is None: @@ -67,6 +68,7 @@ class BookmarkFactoryMixin: shared=shared, web_archive_snapshot_url=web_archive_snapshot_url, favicon_file=favicon_file, + preview_image_file=preview_image_file, ) bookmark.save() for tag in tags: diff --git a/bookmarks/tests/test_bookmark_details_modal.py b/bookmarks/tests/test_bookmark_details_modal.py index abe529b..d24e2d9 100644 --- a/bookmarks/tests/test_bookmark_details_modal.py +++ b/bookmarks/tests/test_bookmark_details_modal.py @@ -300,6 +300,36 @@ class BookmarkDetailsModalTestCase(TestCase, BookmarkFactoryMixin, HtmlTestMixin web_archive_link["target"], UserProfile.BOOKMARK_LINK_TARGET_SELF ) + def test_preview_image(self): + # without image + bookmark = self.setup_bookmark() + soup = self.get_details(bookmark) + image = soup.select_one("div.preview-image img") + self.assertIsNone(image) + + # with image + bookmark = self.setup_bookmark(preview_image_file="example.png") + soup = self.get_details(bookmark) + image = soup.select_one("div.preview-image img") + self.assertIsNone(image) + + # preview images enabled, no image + profile = self.get_or_create_test_user().profile + profile.enable_preview_images = True + profile.save() + + bookmark = self.setup_bookmark() + soup = self.get_details(bookmark) + image = soup.select_one("div.preview-image img") + self.assertIsNone(image) + + # preview images enabled, image present + bookmark = self.setup_bookmark(preview_image_file="example.png") + soup = self.get_details(bookmark) + image = soup.select_one("div.preview-image img") + self.assertIsNotNone(image) + self.assertEqual(image["src"], "/static/example.png") + def test_status(self): # renders form bookmark = self.setup_bookmark() diff --git a/bookmarks/tests/test_bookmarks_api.py b/bookmarks/tests/test_bookmarks_api.py index dae6754..766f3eb 100644 --- a/bookmarks/tests/test_bookmarks_api.py +++ b/bookmarks/tests/test_bookmarks_api.py @@ -628,7 +628,10 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin): website_loader, "load_website_metadata" ) as mock_load_website_metadata: expected_metadata = WebsiteMetadata( - "https://example.com", "Scraped metadata", "Scraped description" + "https://example.com", + "Scraped metadata", + "Scraped description", + "https://example.com/preview.png", ) mock_load_website_metadata.return_value = expected_metadata @@ -640,9 +643,10 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin): metadata = response.data["metadata"] self.assertIsNotNone(metadata) - self.assertIsNotNone(expected_metadata.url, metadata["url"]) - self.assertIsNotNone(expected_metadata.title, metadata["title"]) - self.assertIsNotNone(expected_metadata.description, metadata["description"]) + self.assertEqual(expected_metadata.url, metadata["url"]) + self.assertEqual(expected_metadata.title, metadata["title"]) + self.assertEqual(expected_metadata.description, metadata["description"]) + self.assertEqual(expected_metadata.preview_image, metadata["preview_image"]) def test_check_returns_bookmark_if_url_is_bookmarked(self): self.authenticate() @@ -687,9 +691,10 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin): mock_load_website_metadata.assert_not_called() self.assertIsNotNone(metadata) - self.assertIsNotNone(bookmark.url, metadata["url"]) - self.assertIsNotNone(bookmark.website_title, metadata["title"]) - self.assertIsNotNone(bookmark.website_description, metadata["description"]) + self.assertEqual(bookmark.url, metadata["url"]) + self.assertEqual(bookmark.website_title, metadata["title"]) + self.assertEqual(bookmark.website_description, metadata["description"]) + self.assertIsNone(metadata["preview_image"]) def test_can_only_access_own_bookmarks(self): self.authenticate() diff --git a/bookmarks/tests/test_bookmarks_list_template.py b/bookmarks/tests/test_bookmarks_list_template.py index 6e15ee2..dc7522e 100644 --- a/bookmarks/tests/test_bookmarks_list_template.py +++ b/bookmarks/tests/test_bookmarks_list_template.py @@ -20,7 +20,7 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin): self, html: str, bookmark: Bookmark, link_target: str = "_blank" ): favicon_img = ( - f'' + f'' if bookmark.favicon_file else "" ) @@ -148,19 +148,41 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin): ) def assertFaviconVisible(self, html: str, bookmark: Bookmark): - self.assertFaviconCount(html, bookmark, 1) + self.assertFavicon(html, bookmark, True) def assertFaviconHidden(self, html: str, bookmark: Bookmark): - self.assertFaviconCount(html, bookmark, 0) + self.assertFavicon(html, bookmark, False) - def assertFaviconCount(self, html: str, bookmark: Bookmark, count=1): - self.assertInHTML( - f""" - - """, - html, - count=count, - ) + def assertFavicon(self, html: str, bookmark: Bookmark, visible=True): + soup = self.make_soup(html) + + favicon = soup.select_one(".favicon") + + if not visible: + self.assertIsNone(favicon) + return + + url = f"/static/{bookmark.favicon_file}" + self.assertIsNotNone(favicon) + self.assertEqual(favicon["src"], url) + + def assertPreviewImageVisible(self, html: str, bookmark: Bookmark): + self.assertPreviewImage(html, bookmark, True) + + def assertPreviewImageHidden(self, html: str, bookmark: Bookmark): + self.assertPreviewImage(html, bookmark, False) + + def assertPreviewImage(self, html: str, bookmark: Bookmark, visible=True): + soup = self.make_soup(html) + preview_image = soup.select_one(".preview-image") + + if not visible: + self.assertIsNone(preview_image) + return + + url = f"/static/{bookmark.preview_image_file}" + self.assertIsNotNone(preview_image) + self.assertEqual(preview_image["src"], url) def assertBookmarkURLCount( self, html: str, bookmark: Bookmark, link_target: str = "_blank", count=0 @@ -640,6 +662,36 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin): html, ) + def test_preview_image_should_be_visible_when_preview_images_enabled(self): + profile = self.get_or_create_test_user().profile + profile.enable_preview_images = True + profile.save() + + bookmark = self.setup_bookmark(preview_image_file="preview.png") + html = self.render_template() + + self.assertPreviewImageVisible(html, bookmark) + + def test_preview_image_should_be_hidden_when_preview_images_disabled(self): + profile = self.get_or_create_test_user().profile + profile.enable_preview_images = False + profile.save() + + bookmark = self.setup_bookmark(preview_image_file="preview.png") + html = self.render_template() + + self.assertPreviewImageHidden(html, bookmark) + + def test_preview_image_should_be_hidden_when_there_is_no_preview_image(self): + profile = self.get_or_create_test_user().profile + profile.enable_preview_images = True + profile.save() + + bookmark = self.setup_bookmark() + html = self.render_template() + + self.assertPreviewImageHidden(html, bookmark) + def test_favicon_should_be_visible_when_favicons_enabled(self): profile = self.get_or_create_test_user().profile profile.enable_favicons = True diff --git a/bookmarks/tests/test_bookmarks_service.py b/bookmarks/tests/test_bookmarks_service.py index 5326d9c..b251cce 100644 --- a/bookmarks/tests/test_bookmarks_service.py +++ b/bookmarks/tests/test_bookmarks_service.py @@ -42,7 +42,10 @@ class BookmarkServiceTestCase(TestCase, BookmarkFactoryMixin): website_loader, "load_website_metadata" ) as mock_load_website_metadata: expected_metadata = WebsiteMetadata( - "https://example.com", "Website title", "Website description" + "https://example.com", + "Website title", + "Website description", + "https://example.com/preview.png", ) mock_load_website_metadata.return_value = expected_metadata @@ -157,6 +160,7 @@ class BookmarkServiceTestCase(TestCase, BookmarkFactoryMixin): "https://example.com/updated", "Updated website title", "Updated website description", + "https://example.com/preview.png", ) mock_load_website_metadata.return_value = expected_metadata diff --git a/bookmarks/tests/test_bookmarks_tasks.py b/bookmarks/tests/test_bookmarks_tasks.py index 3c087d7..1376285 100644 --- a/bookmarks/tests/test_bookmarks_tasks.py +++ b/bookmarks/tests/test_bookmarks_tasks.py @@ -74,11 +74,18 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin): self.mock_singlefile_create_snapshot_patcher.start() ) + self.mock_load_preview_image_patcher = mock.patch( + "bookmarks.services.preview_image_loader.load_preview_image" + ) + self.mock_load_preview_image = self.mock_load_preview_image_patcher.start() + self.mock_load_preview_image.return_value = "preview_image.png" + user = self.get_or_create_test_user() user.profile.web_archive_integration = ( UserProfile.WEB_ARCHIVE_INTEGRATION_ENABLED ) user.profile.enable_favicons = True + user.profile.enable_preview_images = True user.profile.save() def tearDown(self): @@ -86,6 +93,7 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin): self.mock_cdx_api_patcher.stop() self.mock_load_favicon_patcher.stop() self.mock_singlefile_create_snapshot_patcher.stop() + self.mock_load_preview_image_patcher.stop() huey.storage.flush_results() huey.immediate = False @@ -507,6 +515,69 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin): self.assertEqual(self.executed_count(), 0) + def test_load_preview_image_should_create_preview_image_file(self): + bookmark = self.setup_bookmark() + + tasks.load_preview_image(self.get_or_create_test_user(), bookmark) + bookmark.refresh_from_db() + + self.assertEqual(self.executed_count(), 1) + self.assertEqual(bookmark.preview_image_file, "preview_image.png") + + def test_load_preview_image_should_update_preview_image_file(self): + bookmark = self.setup_bookmark( + preview_image_file="preview_image.png", + ) + + self.mock_load_preview_image.return_value = "preview_image_upd.png" + + tasks.load_preview_image(self.get_or_create_test_user(), bookmark) + + bookmark.refresh_from_db() + self.mock_load_preview_image.assert_called_once() + self.assertEqual(bookmark.preview_image_file, "preview_image_upd.png") + + def test_load_preview_image_should_handle_missing_bookmark(self): + tasks._load_preview_image_task(123) + + self.mock_load_preview_image.assert_not_called() + + def test_load_preview_image_should_not_save_stale_bookmark_data(self): + bookmark = self.setup_bookmark() + + # update bookmark during API call to check that saving + # the image does not overwrite updated bookmark data + def mock_load_preview_image_impl(url): + bookmark.title = "Updated title" + bookmark.save() + return "test.png" + + self.mock_load_preview_image.side_effect = mock_load_preview_image_impl + + tasks.load_preview_image(self.get_or_create_test_user(), bookmark) + bookmark.refresh_from_db() + + self.assertEqual(bookmark.title, "Updated title") + self.assertEqual(bookmark.preview_image_file, "test.png") + + @override_settings(LD_DISABLE_BACKGROUND_TASKS=True) + def test_load_preview_image_should_not_run_when_background_tasks_are_disabled(self): + bookmark = self.setup_bookmark() + tasks.load_preview_image(self.get_or_create_test_user(), bookmark) + + self.assertEqual(self.executed_count(), 0) + + def test_load_preview_image_should_not_run_when_preview_image_feature_is_disabled( + self, + ): + self.user.profile.enable_preview_images = False + self.user.profile.save() + + bookmark = self.setup_bookmark() + tasks.load_preview_image(self.get_or_create_test_user(), bookmark) + + self.assertEqual(self.executed_count(), 0) + @override_settings(LD_ENABLE_SNAPSHOTS=True) def test_create_html_snapshot_should_create_pending_asset(self): bookmark = self.setup_bookmark() diff --git a/bookmarks/tests/test_preview_image_loader.py b/bookmarks/tests/test_preview_image_loader.py new file mode 100644 index 0000000..66dcd6c --- /dev/null +++ b/bookmarks/tests/test_preview_image_loader.py @@ -0,0 +1,108 @@ +import io +import os +import tempfile +from pathlib import Path +from unittest import mock + +from django.conf import settings +from django.test import TestCase + +from bookmarks.services import preview_image_loader + +mock_image_data = b"mock_image" + + +class MockStreamingResponse: + def __init__(self, data=mock_image_data, content_type="image/png"): + self.chunks = [data] + self.headers = {"Content-Type": content_type} + + def iter_content(self, **kwargs): + return self.chunks + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + pass + + +class PreviewImageLoaderTestCase(TestCase): + def setUp(self) -> None: + self.temp_folder = tempfile.TemporaryDirectory() + self.settings_override = self.settings(LD_PREVIEW_FOLDER=self.temp_folder.name) + self.settings_override.enable() + self.mock_load_website_metadata_patcher = mock.patch( + "bookmarks.services.website_loader.load_website_metadata" + ) + self.mock_load_website_metadata = ( + self.mock_load_website_metadata_patcher.start() + ) + self.mock_load_website_metadata.return_value = mock.Mock( + preview_image="https://example.com/image.png" + ) + + def tearDown(self) -> None: + self.temp_folder.cleanup() + self.settings_override.disable() + self.mock_load_website_metadata_patcher.stop() + + def create_mock_response(self, icon_data=mock_image_data, content_type="image/png"): + mock_response = mock.Mock() + mock_response.raw = io.BytesIO(icon_data) + return MockStreamingResponse(icon_data, content_type) + + def get_image_path(self, filename): + return Path(os.path.join(settings.LD_PREVIEW_FOLDER, filename)) + + def image_exists(self, filename): + return self.get_image_path(filename).exists() + + def get_image_data(self, filename): + return self.get_image_path(filename).read_bytes() + + def test_load_preview_image(self): + with mock.patch("requests.get") as mock_get: + mock_get.return_value = self.create_mock_response() + + file = preview_image_loader.load_preview_image("https://example.com") + + self.assertTrue(self.image_exists(file)) + self.assertEqual(mock_image_data, self.get_image_data(file)) + + def test_load_preview_image_returns_none_if_no_preview_image_detected(self): + with mock.patch("requests.get") as mock_get: + mock_get.return_value = self.create_mock_response() + self.mock_load_website_metadata.return_value = mock.Mock(preview_image=None) + + file = preview_image_loader.load_preview_image("https://example.com") + + self.assertIsNone(file) + + def test_load_preview_image_creates_folder_if_not_exists(self): + with mock.patch("requests.get") as mock_get: + mock_get.return_value = self.create_mock_response() + + folder = Path(settings.LD_PREVIEW_FOLDER) + folder.rmdir() + + self.assertFalse(folder.exists()) + + preview_image_loader.load_preview_image("https://example.com") + + self.assertTrue(folder.exists()) + + def test_guess_file_extension(self): + with mock.patch("requests.get") as mock_get: + mock_get.return_value = self.create_mock_response(content_type="image/png") + file = preview_image_loader.load_preview_image("https://example.com") + + self.assertTrue(self.image_exists(file)) + self.assertEqual("png", file.split(".")[-1]) + + with mock.patch("requests.get") as mock_get: + mock_get.return_value = self.create_mock_response(content_type="image/jpeg") + file = preview_image_loader.load_preview_image("https://example.com") + + self.assertTrue(self.image_exists(file)) + self.assertEqual("jpg", file.split(".")[-1]) diff --git a/bookmarks/tests/test_website_loader.py b/bookmarks/tests/test_website_loader.py index b14ccc4..bfb39c8 100644 --- a/bookmarks/tests/test_website_loader.py +++ b/bookmarks/tests/test_website_loader.py @@ -29,7 +29,9 @@ class WebsiteLoaderTestCase(TestCase): # clear cached metadata before test run website_loader.load_website_metadata.cache_clear() - def render_html_document(self, title, description="", og_description=""): + def render_html_document( + self, title, description="", og_description="", og_image="" + ): meta_description = ( f'' if description else "" ) @@ -38,6 +40,9 @@ class WebsiteLoaderTestCase(TestCase): if og_description else "" ) + meta_og_image = ( + f'' if og_image else "" + ) return f""" @@ -46,6 +51,7 @@ class WebsiteLoaderTestCase(TestCase): {title} {meta_description} {meta_og_description} + {meta_og_image} @@ -105,6 +111,7 @@ class WebsiteLoaderTestCase(TestCase): metadata = website_loader.load_website_metadata("https://example.com") self.assertEqual("test title", metadata.title) self.assertEqual("test description", metadata.description) + self.assertIsNone(metadata.preview_image) def test_load_website_metadata_trims_title_and_description(self): with mock.patch( @@ -128,6 +135,44 @@ class WebsiteLoaderTestCase(TestCase): self.assertEqual("test title", metadata.title) self.assertEqual("test og description", metadata.description) + def test_load_website_metadata_using_og_image(self): + with mock.patch( + "bookmarks.services.website_loader.load_page" + ) as mock_load_page: + mock_load_page.return_value = self.render_html_document( + "test title", og_image="http://example.com/image.jpg" + ) + metadata = website_loader.load_website_metadata("https://example.com") + self.assertEqual("http://example.com/image.jpg", metadata.preview_image) + + def test_load_website_metadata_gets_absolute_og_image_path_when_path_starts_with_dots( + self, + ): + with mock.patch( + "bookmarks.services.website_loader.load_page" + ) as mock_load_page: + mock_load_page.return_value = self.render_html_document( + "test title", og_image="../image.jpg" + ) + metadata = website_loader.load_website_metadata( + "https://example.com/a/b/page.html" + ) + self.assertEqual("https://example.com/a/image.jpg", metadata.preview_image) + + def test_load_website_metadata_gets_absolute_og_image_path_when_path_starts_with_slash( + self, + ): + with mock.patch( + "bookmarks.services.website_loader.load_page" + ) as mock_load_page: + mock_load_page.return_value = self.render_html_document( + "test title", og_image="/image.jpg" + ) + metadata = website_loader.load_website_metadata( + "https://example.com/a/b/page.html" + ) + self.assertEqual("https://example.com/image.jpg", metadata.preview_image) + def test_load_website_metadata_prefers_description_over_og_description(self): with mock.patch( "bookmarks.services.website_loader.load_page" diff --git a/bookmarks/views/partials/contexts.py b/bookmarks/views/partials/contexts.py index 53821f1..722f343 100644 --- a/bookmarks/views/partials/contexts.py +++ b/bookmarks/views/partials/contexts.py @@ -145,6 +145,7 @@ class BookmarkItem: self.tag_names = bookmark.tag_names self.web_archive_snapshot_url = bookmark.web_archive_snapshot_url self.favicon_file = bookmark.favicon_file + self.preview_image_file = bookmark.preview_image_file self.is_archived = bookmark.is_archived self.unread = bookmark.unread self.owner = bookmark.owner @@ -215,6 +216,7 @@ class BookmarkListContext: self.show_archive_action = user_profile.display_archive_bookmark_action self.show_remove_action = user_profile.display_remove_bookmark_action self.show_favicons = user_profile.enable_favicons + self.show_preview_images = user_profile.enable_preview_images self.show_notes = user_profile.permanent_notes @staticmethod @@ -384,6 +386,7 @@ class BookmarkDetailsContext: self.profile = request.user_profile self.is_editable = bookmark.owner == user self.sharing_enabled = user_profile.enable_sharing + self.preview_image_enabled = user_profile.enable_preview_images self.show_link_icons = user_profile.enable_favicons and bookmark.favicon_file # For now hide files section if snapshots are not supported self.show_files = settings.LD_ENABLE_SNAPSHOTS diff --git a/bootstrap.sh b/bootstrap.sh index 4da3861..a3d728f 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -7,6 +7,8 @@ LD_SERVER_PORT="${LD_SERVER_PORT:-9090}" mkdir -p data # Create favicon folder if it does not exist mkdir -p data/favicons +# Create previews folder if it does not exist +mkdir -p data/previews # Create assets folder if it does not exist mkdir -p data/assets diff --git a/package-lock.json b/package-lock.json index aaf0ebb..899a253 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "linkding", - "version": "1.25.0", + "version": "1.30.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "linkding", - "version": "1.25.0", + "version": "1.30.0", "license": "MIT", "dependencies": { "@rollup/plugin-node-resolve": "^15.2.3", diff --git a/siteroot/settings/base.py b/siteroot/settings/base.py index f98b9d5..3f3ad59 100644 --- a/siteroot/settings/base.py +++ b/siteroot/settings/base.py @@ -144,6 +144,7 @@ STATICFILES_FINDERS = [ STATICFILES_DIRS = [ os.path.join(BASE_DIR, "bookmarks", "styles"), os.path.join(BASE_DIR, "data", "favicons"), + os.path.join(BASE_DIR, "data", "previews"), ] # REST framework @@ -286,6 +287,9 @@ LD_ENABLE_REFRESH_FAVICONS = os.getenv("LD_ENABLE_REFRESH_FAVICONS", True) in ( "1", ) +# Previews settings +LD_PREVIEW_FOLDER = os.path.join(BASE_DIR, "data", "previews") + # Asset / snapshot settings LD_ASSET_FOLDER = os.path.join(BASE_DIR, "data", "assets") diff --git a/uwsgi.ini b/uwsgi.ini index b398166..a8a6edd 100644 --- a/uwsgi.ini +++ b/uwsgi.ini @@ -3,6 +3,7 @@ module = siteroot.wsgi:application env = DJANGO_SETTINGS_MODULE=siteroot.settings.prod static-map = /static=static static-map = /static=data/favicons +static-map = /static=data/previews processes = 2 threads = 2 pidfile = /tmp/linkding.pid @@ -16,6 +17,7 @@ die-on-term = true if-env = LD_CONTEXT_PATH static-map = /%(_)static=static static-map = /%(_)static=data/favicons +static-map = /%(_)static=data/previews endif = if-env = LD_REQUEST_TIMEOUT