mirror of
https://github.com/sissbruecker/linkding
synced 2024-11-10 06:04:15 +00:00
937858cf58
* Avoid stall on web scraping This patch fixes stall on web scraping. I encountered a stall (scraping never ends) when adding a bookmark of some site. To avoid this case, adding a timeout parameter at requests.get() function is a solution. Signed-off-by: Taku Izumi <admin@orz-style.com> * Avoid character corruption of scraping some Japanese sites This patch fixes character corruption of scraping some Japanese sites. To avoid character corruption, I use r.content instead of r.text in load_page function. The reason of character corruption is encoding problem, I think. r.text handles data as unicode encoded text, so if scraping web site's charset is not unicode encoded, character corruption occurs. r.content handles data as str[], we can avoid encoding problem. Signed-off-by: Taku Izumi <admin@orz-style.com> * use charset_normalizer to determine response encoding Co-authored-by: Taku Izumi <admin@orz-style.com> Co-authored-by: Sascha Ißbrücker <sascha.issbruecker@googlemail.com>
44 lines
1.3 KiB
Python
44 lines
1.3 KiB
Python
from dataclasses import dataclass
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from charset_normalizer import from_bytes
|
|
|
|
|
|
@dataclass
|
|
class WebsiteMetadata:
|
|
url: str
|
|
title: str
|
|
description: str
|
|
|
|
def to_dict(self):
|
|
return {
|
|
'url': self.url,
|
|
'title': self.title,
|
|
'description': self.description,
|
|
}
|
|
|
|
|
|
def load_website_metadata(url: str):
|
|
title = None
|
|
description = None
|
|
try:
|
|
page_text = load_page(url)
|
|
soup = BeautifulSoup(page_text, 'html.parser')
|
|
|
|
title = soup.title.string if soup.title is not None else None
|
|
description_tag = soup.find('meta', attrs={'name': 'description'})
|
|
description = description_tag['content'] if description_tag is not None else None
|
|
finally:
|
|
return WebsiteMetadata(url=url, title=title, description=description)
|
|
|
|
|
|
def load_page(url: str):
|
|
r = requests.get(url, timeout=10)
|
|
|
|
# Use charset_normalizer to determine encoding that best matches the response content
|
|
# Several sites seem to specify the response encoding incorrectly, so we ignore it and use custom logic instead
|
|
# This is different from Response.text which does respect the encoding specified in the response first,
|
|
# before trying to determine one
|
|
results = from_bytes(r.content)
|
|
return str(results.best())
|