mirror of
https://github.com/sissbruecker/linkding
synced 2024-11-10 06:04:15 +00:00
Add support for bookmark thumbnails (#721)
* Preview Image * fix tests * add test * download preview image * relative path * gst * details view * fix tests * Improve preview image styles * Remove preview image URL from model * Revert form changes * update tests * make it work in uwsgi --------- Co-authored-by: Sascha Ißbrücker <sascha.issbruecker@gmail.com>
This commit is contained in:
parent
e2415f652b
commit
87cd4061cb
26 changed files with 640 additions and 147 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -194,3 +194,5 @@ typings/
|
|||
# ublock + chromium
|
||||
/uBlock0.chromium
|
||||
/chromium-profile
|
||||
# direnv
|
||||
/.direnv
|
||||
|
|
|
@ -99,7 +99,10 @@ class BookmarkViewSet(
|
|||
# Either return metadata from existing bookmark, or scrape from URL
|
||||
if bookmark:
|
||||
metadata = WebsiteMetadata(
|
||||
url, bookmark.website_title, bookmark.website_description
|
||||
url,
|
||||
bookmark.website_title,
|
||||
bookmark.website_description,
|
||||
None,
|
||||
)
|
||||
else:
|
||||
metadata = website_loader.load_website_metadata(url)
|
||||
|
|
|
@ -0,0 +1,23 @@
|
|||
# Generated by Django 5.0.3 on 2024-05-07 07:27
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
("bookmarks", "0033_userprofile_default_mark_unread"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="bookmark",
|
||||
name="preview_image_file",
|
||||
field=models.CharField(blank=True, max_length=512, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="userprofile",
|
||||
name="enable_preview_images",
|
||||
field=models.BooleanField(default=False),
|
||||
),
|
||||
]
|
|
@ -59,6 +59,7 @@ class Bookmark(models.Model):
|
|||
website_description = models.TextField(blank=True, null=True)
|
||||
web_archive_snapshot_url = models.CharField(max_length=2048, blank=True)
|
||||
favicon_file = models.CharField(max_length=512, blank=True)
|
||||
preview_image_file = models.CharField(max_length=512, blank=True, null=True)
|
||||
unread = models.BooleanField(default=False)
|
||||
is_archived = models.BooleanField(default=False)
|
||||
shared = models.BooleanField(default=False)
|
||||
|
@ -394,6 +395,7 @@ class UserProfile(models.Model):
|
|||
enable_sharing = models.BooleanField(default=False, null=False)
|
||||
enable_public_sharing = models.BooleanField(default=False, null=False)
|
||||
enable_favicons = models.BooleanField(default=False, null=False)
|
||||
enable_preview_images = models.BooleanField(default=False, null=False)
|
||||
display_url = models.BooleanField(default=False, null=False)
|
||||
display_view_bookmark_action = models.BooleanField(default=True, null=False)
|
||||
display_edit_bookmark_action = models.BooleanField(default=True, null=False)
|
||||
|
@ -420,6 +422,7 @@ class UserProfileForm(forms.ModelForm):
|
|||
"enable_sharing",
|
||||
"enable_public_sharing",
|
||||
"enable_favicons",
|
||||
"enable_preview_images",
|
||||
"enable_automatic_html_snapshots",
|
||||
"display_url",
|
||||
"display_view_bookmark_action",
|
||||
|
|
|
@ -40,6 +40,8 @@ def create_bookmark(bookmark: Bookmark, tag_string: str, current_user: User):
|
|||
tasks.create_web_archive_snapshot(current_user, bookmark, False)
|
||||
# Load favicon
|
||||
tasks.load_favicon(current_user, bookmark)
|
||||
# Load preview image
|
||||
tasks.load_preview_image(current_user, bookmark)
|
||||
# Create HTML snapshot
|
||||
if current_user.profile.enable_automatic_html_snapshots:
|
||||
tasks.create_html_snapshot(bookmark)
|
||||
|
@ -58,6 +60,8 @@ def update_bookmark(bookmark: Bookmark, tag_string, current_user: User):
|
|||
bookmark.save()
|
||||
# Update favicon
|
||||
tasks.load_favicon(current_user, bookmark)
|
||||
# Update preview image
|
||||
tasks.load_preview_image(current_user, bookmark)
|
||||
|
||||
if has_url_changed:
|
||||
# Update web archive snapshot, if URL changed
|
||||
|
|
46
bookmarks/services/preview_image_loader.py
Normal file
46
bookmarks/services/preview_image_loader.py
Normal file
|
@ -0,0 +1,46 @@
|
|||
import logging
|
||||
import mimetypes
|
||||
import os.path
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from django.conf import settings
|
||||
from bookmarks.services import website_loader
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _ensure_preview_folder():
|
||||
Path(settings.LD_PREVIEW_FOLDER).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def _url_to_filename(preview_image: str) -> str:
|
||||
return hashlib.md5(preview_image.encode()).hexdigest()
|
||||
|
||||
|
||||
def _get_image_path(preview_image_file: str) -> Path:
|
||||
return Path(os.path.join(settings.LD_PREVIEW_FOLDER, preview_image_file))
|
||||
|
||||
|
||||
def load_preview_image(url: str) -> str | None:
|
||||
_ensure_preview_folder()
|
||||
|
||||
metadata = website_loader.load_website_metadata(url)
|
||||
if not metadata.preview_image:
|
||||
logger.debug(f"Could not find preview image in metadata: {url}")
|
||||
return None
|
||||
|
||||
logger.debug(f"Loading preview image: {metadata.preview_image}")
|
||||
with requests.get(metadata.preview_image, stream=True) as response:
|
||||
content_type = response.headers["Content-Type"]
|
||||
preview_image_hash = _url_to_filename(url)
|
||||
file_extension = mimetypes.guess_extension(content_type)
|
||||
preview_image_file = f"{preview_image_hash}{file_extension}"
|
||||
preview_image_path = _get_image_path(preview_image_file)
|
||||
with open(preview_image_path, "wb") as file:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
file.write(chunk)
|
||||
logger.debug(f"Saved preview image as: {preview_image_path}")
|
||||
|
||||
return preview_image_file
|
|
@ -15,7 +15,7 @@ from waybackpy.exceptions import WaybackError, TooManyRequestsError, NoCDXRecord
|
|||
|
||||
import bookmarks.services.wayback
|
||||
from bookmarks.models import Bookmark, BookmarkAsset, UserProfile
|
||||
from bookmarks.services import favicon_loader, singlefile
|
||||
from bookmarks.services import favicon_loader, singlefile, preview_image_loader
|
||||
from bookmarks.services.website_loader import DEFAULT_USER_AGENT
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
@ -221,6 +221,30 @@ def _schedule_refresh_favicons_task(user_id: int):
|
|||
_load_favicon_task(bookmark.id)
|
||||
|
||||
|
||||
def load_preview_image(user: User, bookmark: Bookmark):
|
||||
if user.profile.enable_preview_images and not settings.LD_DISABLE_BACKGROUND_TASKS:
|
||||
_load_preview_image_task(bookmark.id)
|
||||
|
||||
|
||||
@task()
|
||||
def _load_preview_image_task(bookmark_id: int):
|
||||
try:
|
||||
bookmark = Bookmark.objects.get(id=bookmark_id)
|
||||
except Bookmark.DoesNotExist:
|
||||
return
|
||||
|
||||
logger.info(f"Load preview image for bookmark. url={bookmark.url}")
|
||||
|
||||
new_preview_image_file = preview_image_loader.load_preview_image(bookmark.url)
|
||||
|
||||
if new_preview_image_file != bookmark.preview_image_file:
|
||||
bookmark.preview_image_file = new_preview_image_file
|
||||
bookmark.save(update_fields=["preview_image_file"])
|
||||
logger.info(
|
||||
f"Successfully updated preview image for bookmark. url={bookmark.url} preview_image_file={new_preview_image_file}"
|
||||
)
|
||||
|
||||
|
||||
def is_html_snapshot_feature_active() -> bool:
|
||||
return settings.LD_ENABLE_SNAPSHOTS and not settings.LD_DISABLE_BACKGROUND_TASKS
|
||||
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
import logging
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
@ -15,12 +16,14 @@ class WebsiteMetadata:
|
|||
url: str
|
||||
title: str
|
||||
description: str
|
||||
preview_image: str | None
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"url": self.url,
|
||||
"title": self.title,
|
||||
"description": self.description,
|
||||
"preview_image": self.preview_image,
|
||||
}
|
||||
|
||||
|
||||
|
@ -30,6 +33,7 @@ class WebsiteMetadata:
|
|||
def load_website_metadata(url: str):
|
||||
title = None
|
||||
description = None
|
||||
preview_image = None
|
||||
try:
|
||||
start = timezone.now()
|
||||
page_text = load_page(url)
|
||||
|
@ -55,10 +59,21 @@ def load_website_metadata(url: str):
|
|||
else None
|
||||
)
|
||||
|
||||
image_tag = soup.find("meta", attrs={"property": "og:image"})
|
||||
preview_image = image_tag["content"].strip() if image_tag else None
|
||||
if (
|
||||
preview_image
|
||||
and not preview_image.startswith("http://")
|
||||
and not preview_image.startswith("https://")
|
||||
):
|
||||
preview_image = urljoin(url, preview_image)
|
||||
|
||||
end = timezone.now()
|
||||
logger.debug(f"Parsing duration: {end - start}")
|
||||
finally:
|
||||
return WebsiteMetadata(url=url, title=title, description=description)
|
||||
return WebsiteMetadata(
|
||||
url=url, title=title, description=description, preview_image=preview_image
|
||||
)
|
||||
|
||||
|
||||
CHUNK_SIZE = 50 * 1024
|
||||
|
|
|
@ -33,6 +33,15 @@
|
|||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.preview-image {
|
||||
margin: $unit-4 0;
|
||||
|
||||
img {
|
||||
max-width: 100%;
|
||||
max-height: 200px;
|
||||
}
|
||||
}
|
||||
|
||||
dl {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
|
|
@ -128,8 +128,25 @@ ul.bookmark-list {
|
|||
/* Bookmarks */
|
||||
li[ld-bookmark-item] {
|
||||
position: relative;
|
||||
display: flex;
|
||||
gap: $unit-2;
|
||||
margin-top: $unit-2;
|
||||
|
||||
.content {
|
||||
flex: 1 1 0;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
img.preview-image {
|
||||
flex: 0 0 auto;
|
||||
width: 100px;
|
||||
height: 60px;
|
||||
margin-top: $unit-h;
|
||||
object-fit: cover;
|
||||
border-radius: $border-radius;
|
||||
border: solid 1px $border-color-dark;
|
||||
}
|
||||
|
||||
.form-checkbox.bulk-edit-checkbox {
|
||||
display: none;
|
||||
}
|
||||
|
|
|
@ -10,13 +10,14 @@
|
|||
data-bookmarks-total="{{ bookmark_list.bookmarks_total }}">
|
||||
{% for bookmark_item in bookmark_list.items %}
|
||||
<li ld-bookmark-item{% if bookmark_item.css_classes %} class="{{ bookmark_item.css_classes }}"{% endif %}>
|
||||
<div class="content">
|
||||
<div class="title">
|
||||
<label class="form-checkbox bulk-edit-checkbox">
|
||||
<input type="checkbox" name="bookmark_id" value="{{ bookmark_item.id }}">
|
||||
<i class="form-icon"></i>
|
||||
</label>
|
||||
{% if bookmark_item.favicon_file and bookmark_list.show_favicons %}
|
||||
<img src="{% static bookmark_item.favicon_file %}" alt="">
|
||||
<img class="favicon" src="{% static bookmark_item.favicon_file %}" alt="">
|
||||
{% endif %}
|
||||
<a href="{{ bookmark_item.url }}" target="{{ bookmark_list.link_target }}" rel="noopener">
|
||||
<span>{{ bookmark_item.title }}</span>
|
||||
|
@ -142,6 +143,10 @@
|
|||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% if bookmark_list.show_preview_images and bookmark_item.preview_image_file %}
|
||||
<img class="preview-image" src="{% static bookmark_item.preview_image_file %}"/>
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
|
|
|
@ -37,6 +37,11 @@
|
|||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if details.preview_image_enabled and details.bookmark.preview_image_file %}
|
||||
<div class="preview-image">
|
||||
<img src="{% static details.bookmark.preview_image_file %}"/>
|
||||
</div>
|
||||
{% endif %}
|
||||
<dl class="grid columns-2 columns-sm-1 gap-0">
|
||||
{% if details.is_editable %}
|
||||
<div class="status col-2">
|
||||
|
|
|
@ -127,6 +127,15 @@
|
|||
<button class="btn mt-2" name="refresh_favicons">Refresh Favicons</button>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="{{ form.enable_preview_images.id_for_label }}" class="form-checkbox">
|
||||
{{ form.enable_preview_images }}
|
||||
<i class="form-icon"></i> Enable Preview Images
|
||||
</label>
|
||||
<div class="form-input-hint">
|
||||
Automatically loads preview images for bookmarked websites and displays them next to each bookmark.
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="{{ form.web_archive_integration.id_for_label }}" class="form-label">Internet Archive
|
||||
integration</label>
|
||||
|
|
|
@ -39,6 +39,7 @@ class BookmarkFactoryMixin:
|
|||
website_description: str = "",
|
||||
web_archive_snapshot_url: str = "",
|
||||
favicon_file: str = "",
|
||||
preview_image_file: str = "",
|
||||
added: datetime = None,
|
||||
):
|
||||
if title is None:
|
||||
|
@ -67,6 +68,7 @@ class BookmarkFactoryMixin:
|
|||
shared=shared,
|
||||
web_archive_snapshot_url=web_archive_snapshot_url,
|
||||
favicon_file=favicon_file,
|
||||
preview_image_file=preview_image_file,
|
||||
)
|
||||
bookmark.save()
|
||||
for tag in tags:
|
||||
|
|
|
@ -300,6 +300,36 @@ class BookmarkDetailsModalTestCase(TestCase, BookmarkFactoryMixin, HtmlTestMixin
|
|||
web_archive_link["target"], UserProfile.BOOKMARK_LINK_TARGET_SELF
|
||||
)
|
||||
|
||||
def test_preview_image(self):
|
||||
# without image
|
||||
bookmark = self.setup_bookmark()
|
||||
soup = self.get_details(bookmark)
|
||||
image = soup.select_one("div.preview-image img")
|
||||
self.assertIsNone(image)
|
||||
|
||||
# with image
|
||||
bookmark = self.setup_bookmark(preview_image_file="example.png")
|
||||
soup = self.get_details(bookmark)
|
||||
image = soup.select_one("div.preview-image img")
|
||||
self.assertIsNone(image)
|
||||
|
||||
# preview images enabled, no image
|
||||
profile = self.get_or_create_test_user().profile
|
||||
profile.enable_preview_images = True
|
||||
profile.save()
|
||||
|
||||
bookmark = self.setup_bookmark()
|
||||
soup = self.get_details(bookmark)
|
||||
image = soup.select_one("div.preview-image img")
|
||||
self.assertIsNone(image)
|
||||
|
||||
# preview images enabled, image present
|
||||
bookmark = self.setup_bookmark(preview_image_file="example.png")
|
||||
soup = self.get_details(bookmark)
|
||||
image = soup.select_one("div.preview-image img")
|
||||
self.assertIsNotNone(image)
|
||||
self.assertEqual(image["src"], "/static/example.png")
|
||||
|
||||
def test_status(self):
|
||||
# renders form
|
||||
bookmark = self.setup_bookmark()
|
||||
|
|
|
@ -628,7 +628,10 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin):
|
|||
website_loader, "load_website_metadata"
|
||||
) as mock_load_website_metadata:
|
||||
expected_metadata = WebsiteMetadata(
|
||||
"https://example.com", "Scraped metadata", "Scraped description"
|
||||
"https://example.com",
|
||||
"Scraped metadata",
|
||||
"Scraped description",
|
||||
"https://example.com/preview.png",
|
||||
)
|
||||
mock_load_website_metadata.return_value = expected_metadata
|
||||
|
||||
|
@ -640,9 +643,10 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin):
|
|||
metadata = response.data["metadata"]
|
||||
|
||||
self.assertIsNotNone(metadata)
|
||||
self.assertIsNotNone(expected_metadata.url, metadata["url"])
|
||||
self.assertIsNotNone(expected_metadata.title, metadata["title"])
|
||||
self.assertIsNotNone(expected_metadata.description, metadata["description"])
|
||||
self.assertEqual(expected_metadata.url, metadata["url"])
|
||||
self.assertEqual(expected_metadata.title, metadata["title"])
|
||||
self.assertEqual(expected_metadata.description, metadata["description"])
|
||||
self.assertEqual(expected_metadata.preview_image, metadata["preview_image"])
|
||||
|
||||
def test_check_returns_bookmark_if_url_is_bookmarked(self):
|
||||
self.authenticate()
|
||||
|
@ -687,9 +691,10 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin):
|
|||
|
||||
mock_load_website_metadata.assert_not_called()
|
||||
self.assertIsNotNone(metadata)
|
||||
self.assertIsNotNone(bookmark.url, metadata["url"])
|
||||
self.assertIsNotNone(bookmark.website_title, metadata["title"])
|
||||
self.assertIsNotNone(bookmark.website_description, metadata["description"])
|
||||
self.assertEqual(bookmark.url, metadata["url"])
|
||||
self.assertEqual(bookmark.website_title, metadata["title"])
|
||||
self.assertEqual(bookmark.website_description, metadata["description"])
|
||||
self.assertIsNone(metadata["preview_image"])
|
||||
|
||||
def test_can_only_access_own_bookmarks(self):
|
||||
self.authenticate()
|
||||
|
|
|
@ -20,7 +20,7 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
|
|||
self, html: str, bookmark: Bookmark, link_target: str = "_blank"
|
||||
):
|
||||
favicon_img = (
|
||||
f'<img src="/static/{bookmark.favicon_file}" alt="">'
|
||||
f'<img class="favicon" src="/static/{bookmark.favicon_file}" alt="">'
|
||||
if bookmark.favicon_file
|
||||
else ""
|
||||
)
|
||||
|
@ -148,19 +148,41 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
|
|||
)
|
||||
|
||||
def assertFaviconVisible(self, html: str, bookmark: Bookmark):
|
||||
self.assertFaviconCount(html, bookmark, 1)
|
||||
self.assertFavicon(html, bookmark, True)
|
||||
|
||||
def assertFaviconHidden(self, html: str, bookmark: Bookmark):
|
||||
self.assertFaviconCount(html, bookmark, 0)
|
||||
self.assertFavicon(html, bookmark, False)
|
||||
|
||||
def assertFaviconCount(self, html: str, bookmark: Bookmark, count=1):
|
||||
self.assertInHTML(
|
||||
f"""
|
||||
<img src="/static/{bookmark.favicon_file}" alt="">
|
||||
""",
|
||||
html,
|
||||
count=count,
|
||||
)
|
||||
def assertFavicon(self, html: str, bookmark: Bookmark, visible=True):
|
||||
soup = self.make_soup(html)
|
||||
|
||||
favicon = soup.select_one(".favicon")
|
||||
|
||||
if not visible:
|
||||
self.assertIsNone(favicon)
|
||||
return
|
||||
|
||||
url = f"/static/{bookmark.favicon_file}"
|
||||
self.assertIsNotNone(favicon)
|
||||
self.assertEqual(favicon["src"], url)
|
||||
|
||||
def assertPreviewImageVisible(self, html: str, bookmark: Bookmark):
|
||||
self.assertPreviewImage(html, bookmark, True)
|
||||
|
||||
def assertPreviewImageHidden(self, html: str, bookmark: Bookmark):
|
||||
self.assertPreviewImage(html, bookmark, False)
|
||||
|
||||
def assertPreviewImage(self, html: str, bookmark: Bookmark, visible=True):
|
||||
soup = self.make_soup(html)
|
||||
preview_image = soup.select_one(".preview-image")
|
||||
|
||||
if not visible:
|
||||
self.assertIsNone(preview_image)
|
||||
return
|
||||
|
||||
url = f"/static/{bookmark.preview_image_file}"
|
||||
self.assertIsNotNone(preview_image)
|
||||
self.assertEqual(preview_image["src"], url)
|
||||
|
||||
def assertBookmarkURLCount(
|
||||
self, html: str, bookmark: Bookmark, link_target: str = "_blank", count=0
|
||||
|
@ -640,6 +662,36 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
|
|||
html,
|
||||
)
|
||||
|
||||
def test_preview_image_should_be_visible_when_preview_images_enabled(self):
|
||||
profile = self.get_or_create_test_user().profile
|
||||
profile.enable_preview_images = True
|
||||
profile.save()
|
||||
|
||||
bookmark = self.setup_bookmark(preview_image_file="preview.png")
|
||||
html = self.render_template()
|
||||
|
||||
self.assertPreviewImageVisible(html, bookmark)
|
||||
|
||||
def test_preview_image_should_be_hidden_when_preview_images_disabled(self):
|
||||
profile = self.get_or_create_test_user().profile
|
||||
profile.enable_preview_images = False
|
||||
profile.save()
|
||||
|
||||
bookmark = self.setup_bookmark(preview_image_file="preview.png")
|
||||
html = self.render_template()
|
||||
|
||||
self.assertPreviewImageHidden(html, bookmark)
|
||||
|
||||
def test_preview_image_should_be_hidden_when_there_is_no_preview_image(self):
|
||||
profile = self.get_or_create_test_user().profile
|
||||
profile.enable_preview_images = True
|
||||
profile.save()
|
||||
|
||||
bookmark = self.setup_bookmark()
|
||||
html = self.render_template()
|
||||
|
||||
self.assertPreviewImageHidden(html, bookmark)
|
||||
|
||||
def test_favicon_should_be_visible_when_favicons_enabled(self):
|
||||
profile = self.get_or_create_test_user().profile
|
||||
profile.enable_favicons = True
|
||||
|
|
|
@ -42,7 +42,10 @@ class BookmarkServiceTestCase(TestCase, BookmarkFactoryMixin):
|
|||
website_loader, "load_website_metadata"
|
||||
) as mock_load_website_metadata:
|
||||
expected_metadata = WebsiteMetadata(
|
||||
"https://example.com", "Website title", "Website description"
|
||||
"https://example.com",
|
||||
"Website title",
|
||||
"Website description",
|
||||
"https://example.com/preview.png",
|
||||
)
|
||||
mock_load_website_metadata.return_value = expected_metadata
|
||||
|
||||
|
@ -157,6 +160,7 @@ class BookmarkServiceTestCase(TestCase, BookmarkFactoryMixin):
|
|||
"https://example.com/updated",
|
||||
"Updated website title",
|
||||
"Updated website description",
|
||||
"https://example.com/preview.png",
|
||||
)
|
||||
mock_load_website_metadata.return_value = expected_metadata
|
||||
|
||||
|
|
|
@ -74,11 +74,18 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
|
|||
self.mock_singlefile_create_snapshot_patcher.start()
|
||||
)
|
||||
|
||||
self.mock_load_preview_image_patcher = mock.patch(
|
||||
"bookmarks.services.preview_image_loader.load_preview_image"
|
||||
)
|
||||
self.mock_load_preview_image = self.mock_load_preview_image_patcher.start()
|
||||
self.mock_load_preview_image.return_value = "preview_image.png"
|
||||
|
||||
user = self.get_or_create_test_user()
|
||||
user.profile.web_archive_integration = (
|
||||
UserProfile.WEB_ARCHIVE_INTEGRATION_ENABLED
|
||||
)
|
||||
user.profile.enable_favicons = True
|
||||
user.profile.enable_preview_images = True
|
||||
user.profile.save()
|
||||
|
||||
def tearDown(self):
|
||||
|
@ -86,6 +93,7 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
|
|||
self.mock_cdx_api_patcher.stop()
|
||||
self.mock_load_favicon_patcher.stop()
|
||||
self.mock_singlefile_create_snapshot_patcher.stop()
|
||||
self.mock_load_preview_image_patcher.stop()
|
||||
huey.storage.flush_results()
|
||||
huey.immediate = False
|
||||
|
||||
|
@ -507,6 +515,69 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
|
|||
|
||||
self.assertEqual(self.executed_count(), 0)
|
||||
|
||||
def test_load_preview_image_should_create_preview_image_file(self):
|
||||
bookmark = self.setup_bookmark()
|
||||
|
||||
tasks.load_preview_image(self.get_or_create_test_user(), bookmark)
|
||||
bookmark.refresh_from_db()
|
||||
|
||||
self.assertEqual(self.executed_count(), 1)
|
||||
self.assertEqual(bookmark.preview_image_file, "preview_image.png")
|
||||
|
||||
def test_load_preview_image_should_update_preview_image_file(self):
|
||||
bookmark = self.setup_bookmark(
|
||||
preview_image_file="preview_image.png",
|
||||
)
|
||||
|
||||
self.mock_load_preview_image.return_value = "preview_image_upd.png"
|
||||
|
||||
tasks.load_preview_image(self.get_or_create_test_user(), bookmark)
|
||||
|
||||
bookmark.refresh_from_db()
|
||||
self.mock_load_preview_image.assert_called_once()
|
||||
self.assertEqual(bookmark.preview_image_file, "preview_image_upd.png")
|
||||
|
||||
def test_load_preview_image_should_handle_missing_bookmark(self):
|
||||
tasks._load_preview_image_task(123)
|
||||
|
||||
self.mock_load_preview_image.assert_not_called()
|
||||
|
||||
def test_load_preview_image_should_not_save_stale_bookmark_data(self):
|
||||
bookmark = self.setup_bookmark()
|
||||
|
||||
# update bookmark during API call to check that saving
|
||||
# the image does not overwrite updated bookmark data
|
||||
def mock_load_preview_image_impl(url):
|
||||
bookmark.title = "Updated title"
|
||||
bookmark.save()
|
||||
return "test.png"
|
||||
|
||||
self.mock_load_preview_image.side_effect = mock_load_preview_image_impl
|
||||
|
||||
tasks.load_preview_image(self.get_or_create_test_user(), bookmark)
|
||||
bookmark.refresh_from_db()
|
||||
|
||||
self.assertEqual(bookmark.title, "Updated title")
|
||||
self.assertEqual(bookmark.preview_image_file, "test.png")
|
||||
|
||||
@override_settings(LD_DISABLE_BACKGROUND_TASKS=True)
|
||||
def test_load_preview_image_should_not_run_when_background_tasks_are_disabled(self):
|
||||
bookmark = self.setup_bookmark()
|
||||
tasks.load_preview_image(self.get_or_create_test_user(), bookmark)
|
||||
|
||||
self.assertEqual(self.executed_count(), 0)
|
||||
|
||||
def test_load_preview_image_should_not_run_when_preview_image_feature_is_disabled(
|
||||
self,
|
||||
):
|
||||
self.user.profile.enable_preview_images = False
|
||||
self.user.profile.save()
|
||||
|
||||
bookmark = self.setup_bookmark()
|
||||
tasks.load_preview_image(self.get_or_create_test_user(), bookmark)
|
||||
|
||||
self.assertEqual(self.executed_count(), 0)
|
||||
|
||||
@override_settings(LD_ENABLE_SNAPSHOTS=True)
|
||||
def test_create_html_snapshot_should_create_pending_asset(self):
|
||||
bookmark = self.setup_bookmark()
|
||||
|
|
108
bookmarks/tests/test_preview_image_loader.py
Normal file
108
bookmarks/tests/test_preview_image_loader.py
Normal file
|
@ -0,0 +1,108 @@
|
|||
import io
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from django.conf import settings
|
||||
from django.test import TestCase
|
||||
|
||||
from bookmarks.services import preview_image_loader
|
||||
|
||||
mock_image_data = b"mock_image"
|
||||
|
||||
|
||||
class MockStreamingResponse:
|
||||
def __init__(self, data=mock_image_data, content_type="image/png"):
|
||||
self.chunks = [data]
|
||||
self.headers = {"Content-Type": content_type}
|
||||
|
||||
def iter_content(self, **kwargs):
|
||||
return self.chunks
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
pass
|
||||
|
||||
|
||||
class PreviewImageLoaderTestCase(TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temp_folder = tempfile.TemporaryDirectory()
|
||||
self.settings_override = self.settings(LD_PREVIEW_FOLDER=self.temp_folder.name)
|
||||
self.settings_override.enable()
|
||||
self.mock_load_website_metadata_patcher = mock.patch(
|
||||
"bookmarks.services.website_loader.load_website_metadata"
|
||||
)
|
||||
self.mock_load_website_metadata = (
|
||||
self.mock_load_website_metadata_patcher.start()
|
||||
)
|
||||
self.mock_load_website_metadata.return_value = mock.Mock(
|
||||
preview_image="https://example.com/image.png"
|
||||
)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.temp_folder.cleanup()
|
||||
self.settings_override.disable()
|
||||
self.mock_load_website_metadata_patcher.stop()
|
||||
|
||||
def create_mock_response(self, icon_data=mock_image_data, content_type="image/png"):
|
||||
mock_response = mock.Mock()
|
||||
mock_response.raw = io.BytesIO(icon_data)
|
||||
return MockStreamingResponse(icon_data, content_type)
|
||||
|
||||
def get_image_path(self, filename):
|
||||
return Path(os.path.join(settings.LD_PREVIEW_FOLDER, filename))
|
||||
|
||||
def image_exists(self, filename):
|
||||
return self.get_image_path(filename).exists()
|
||||
|
||||
def get_image_data(self, filename):
|
||||
return self.get_image_path(filename).read_bytes()
|
||||
|
||||
def test_load_preview_image(self):
|
||||
with mock.patch("requests.get") as mock_get:
|
||||
mock_get.return_value = self.create_mock_response()
|
||||
|
||||
file = preview_image_loader.load_preview_image("https://example.com")
|
||||
|
||||
self.assertTrue(self.image_exists(file))
|
||||
self.assertEqual(mock_image_data, self.get_image_data(file))
|
||||
|
||||
def test_load_preview_image_returns_none_if_no_preview_image_detected(self):
|
||||
with mock.patch("requests.get") as mock_get:
|
||||
mock_get.return_value = self.create_mock_response()
|
||||
self.mock_load_website_metadata.return_value = mock.Mock(preview_image=None)
|
||||
|
||||
file = preview_image_loader.load_preview_image("https://example.com")
|
||||
|
||||
self.assertIsNone(file)
|
||||
|
||||
def test_load_preview_image_creates_folder_if_not_exists(self):
|
||||
with mock.patch("requests.get") as mock_get:
|
||||
mock_get.return_value = self.create_mock_response()
|
||||
|
||||
folder = Path(settings.LD_PREVIEW_FOLDER)
|
||||
folder.rmdir()
|
||||
|
||||
self.assertFalse(folder.exists())
|
||||
|
||||
preview_image_loader.load_preview_image("https://example.com")
|
||||
|
||||
self.assertTrue(folder.exists())
|
||||
|
||||
def test_guess_file_extension(self):
|
||||
with mock.patch("requests.get") as mock_get:
|
||||
mock_get.return_value = self.create_mock_response(content_type="image/png")
|
||||
file = preview_image_loader.load_preview_image("https://example.com")
|
||||
|
||||
self.assertTrue(self.image_exists(file))
|
||||
self.assertEqual("png", file.split(".")[-1])
|
||||
|
||||
with mock.patch("requests.get") as mock_get:
|
||||
mock_get.return_value = self.create_mock_response(content_type="image/jpeg")
|
||||
file = preview_image_loader.load_preview_image("https://example.com")
|
||||
|
||||
self.assertTrue(self.image_exists(file))
|
||||
self.assertEqual("jpg", file.split(".")[-1])
|
|
@ -29,7 +29,9 @@ class WebsiteLoaderTestCase(TestCase):
|
|||
# clear cached metadata before test run
|
||||
website_loader.load_website_metadata.cache_clear()
|
||||
|
||||
def render_html_document(self, title, description="", og_description=""):
|
||||
def render_html_document(
|
||||
self, title, description="", og_description="", og_image=""
|
||||
):
|
||||
meta_description = (
|
||||
f'<meta name="description" content="{description}">' if description else ""
|
||||
)
|
||||
|
@ -38,6 +40,9 @@ class WebsiteLoaderTestCase(TestCase):
|
|||
if og_description
|
||||
else ""
|
||||
)
|
||||
meta_og_image = (
|
||||
f'<meta property="og:image" content="{og_image}">' if og_image else ""
|
||||
)
|
||||
return f"""
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
@ -46,6 +51,7 @@ class WebsiteLoaderTestCase(TestCase):
|
|||
<title>{title}</title>
|
||||
{meta_description}
|
||||
{meta_og_description}
|
||||
{meta_og_image}
|
||||
</head>
|
||||
<body></body>
|
||||
</html>
|
||||
|
@ -105,6 +111,7 @@ class WebsiteLoaderTestCase(TestCase):
|
|||
metadata = website_loader.load_website_metadata("https://example.com")
|
||||
self.assertEqual("test title", metadata.title)
|
||||
self.assertEqual("test description", metadata.description)
|
||||
self.assertIsNone(metadata.preview_image)
|
||||
|
||||
def test_load_website_metadata_trims_title_and_description(self):
|
||||
with mock.patch(
|
||||
|
@ -128,6 +135,44 @@ class WebsiteLoaderTestCase(TestCase):
|
|||
self.assertEqual("test title", metadata.title)
|
||||
self.assertEqual("test og description", metadata.description)
|
||||
|
||||
def test_load_website_metadata_using_og_image(self):
|
||||
with mock.patch(
|
||||
"bookmarks.services.website_loader.load_page"
|
||||
) as mock_load_page:
|
||||
mock_load_page.return_value = self.render_html_document(
|
||||
"test title", og_image="http://example.com/image.jpg"
|
||||
)
|
||||
metadata = website_loader.load_website_metadata("https://example.com")
|
||||
self.assertEqual("http://example.com/image.jpg", metadata.preview_image)
|
||||
|
||||
def test_load_website_metadata_gets_absolute_og_image_path_when_path_starts_with_dots(
|
||||
self,
|
||||
):
|
||||
with mock.patch(
|
||||
"bookmarks.services.website_loader.load_page"
|
||||
) as mock_load_page:
|
||||
mock_load_page.return_value = self.render_html_document(
|
||||
"test title", og_image="../image.jpg"
|
||||
)
|
||||
metadata = website_loader.load_website_metadata(
|
||||
"https://example.com/a/b/page.html"
|
||||
)
|
||||
self.assertEqual("https://example.com/a/image.jpg", metadata.preview_image)
|
||||
|
||||
def test_load_website_metadata_gets_absolute_og_image_path_when_path_starts_with_slash(
|
||||
self,
|
||||
):
|
||||
with mock.patch(
|
||||
"bookmarks.services.website_loader.load_page"
|
||||
) as mock_load_page:
|
||||
mock_load_page.return_value = self.render_html_document(
|
||||
"test title", og_image="/image.jpg"
|
||||
)
|
||||
metadata = website_loader.load_website_metadata(
|
||||
"https://example.com/a/b/page.html"
|
||||
)
|
||||
self.assertEqual("https://example.com/image.jpg", metadata.preview_image)
|
||||
|
||||
def test_load_website_metadata_prefers_description_over_og_description(self):
|
||||
with mock.patch(
|
||||
"bookmarks.services.website_loader.load_page"
|
||||
|
|
|
@ -145,6 +145,7 @@ class BookmarkItem:
|
|||
self.tag_names = bookmark.tag_names
|
||||
self.web_archive_snapshot_url = bookmark.web_archive_snapshot_url
|
||||
self.favicon_file = bookmark.favicon_file
|
||||
self.preview_image_file = bookmark.preview_image_file
|
||||
self.is_archived = bookmark.is_archived
|
||||
self.unread = bookmark.unread
|
||||
self.owner = bookmark.owner
|
||||
|
@ -215,6 +216,7 @@ class BookmarkListContext:
|
|||
self.show_archive_action = user_profile.display_archive_bookmark_action
|
||||
self.show_remove_action = user_profile.display_remove_bookmark_action
|
||||
self.show_favicons = user_profile.enable_favicons
|
||||
self.show_preview_images = user_profile.enable_preview_images
|
||||
self.show_notes = user_profile.permanent_notes
|
||||
|
||||
@staticmethod
|
||||
|
@ -384,6 +386,7 @@ class BookmarkDetailsContext:
|
|||
self.profile = request.user_profile
|
||||
self.is_editable = bookmark.owner == user
|
||||
self.sharing_enabled = user_profile.enable_sharing
|
||||
self.preview_image_enabled = user_profile.enable_preview_images
|
||||
self.show_link_icons = user_profile.enable_favicons and bookmark.favicon_file
|
||||
# For now hide files section if snapshots are not supported
|
||||
self.show_files = settings.LD_ENABLE_SNAPSHOTS
|
||||
|
|
|
@ -7,6 +7,8 @@ LD_SERVER_PORT="${LD_SERVER_PORT:-9090}"
|
|||
mkdir -p data
|
||||
# Create favicon folder if it does not exist
|
||||
mkdir -p data/favicons
|
||||
# Create previews folder if it does not exist
|
||||
mkdir -p data/previews
|
||||
# Create assets folder if it does not exist
|
||||
mkdir -p data/assets
|
||||
|
||||
|
|
4
package-lock.json
generated
4
package-lock.json
generated
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "linkding",
|
||||
"version": "1.25.0",
|
||||
"version": "1.30.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "linkding",
|
||||
"version": "1.25.0",
|
||||
"version": "1.30.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@rollup/plugin-node-resolve": "^15.2.3",
|
||||
|
|
|
@ -144,6 +144,7 @@ STATICFILES_FINDERS = [
|
|||
STATICFILES_DIRS = [
|
||||
os.path.join(BASE_DIR, "bookmarks", "styles"),
|
||||
os.path.join(BASE_DIR, "data", "favicons"),
|
||||
os.path.join(BASE_DIR, "data", "previews"),
|
||||
]
|
||||
|
||||
# REST framework
|
||||
|
@ -286,6 +287,9 @@ LD_ENABLE_REFRESH_FAVICONS = os.getenv("LD_ENABLE_REFRESH_FAVICONS", True) in (
|
|||
"1",
|
||||
)
|
||||
|
||||
# Previews settings
|
||||
LD_PREVIEW_FOLDER = os.path.join(BASE_DIR, "data", "previews")
|
||||
|
||||
# Asset / snapshot settings
|
||||
LD_ASSET_FOLDER = os.path.join(BASE_DIR, "data", "assets")
|
||||
|
||||
|
|
|
@ -3,6 +3,7 @@ module = siteroot.wsgi:application
|
|||
env = DJANGO_SETTINGS_MODULE=siteroot.settings.prod
|
||||
static-map = /static=static
|
||||
static-map = /static=data/favicons
|
||||
static-map = /static=data/previews
|
||||
processes = 2
|
||||
threads = 2
|
||||
pidfile = /tmp/linkding.pid
|
||||
|
@ -16,6 +17,7 @@ die-on-term = true
|
|||
if-env = LD_CONTEXT_PATH
|
||||
static-map = /%(_)static=static
|
||||
static-map = /%(_)static=data/favicons
|
||||
static-map = /%(_)static=data/previews
|
||||
endif =
|
||||
|
||||
if-env = LD_REQUEST_TIMEOUT
|
||||
|
|
Loading…
Reference in a new issue