mirror of
https://github.com/ArchiveBox/ArchiveBox
synced 2024-11-26 14:10:20 +00:00
9f462a87a8
The feedparser packages has 20 years of history and is very good at parsing RSS and Atom, so use that instead of ad-hoc regex and XML parsing. The medium_rss and shaarli_rss parsers weren't touched because they are probably unnecessary. (The special parse for pinboard is just needing because of how tags work.) Doesn't include tests because I haven't figured out how to run them in the docker development setup. Fixes #1171
46 lines
1.1 KiB
Python
46 lines
1.1 KiB
Python
__package__ = 'archivebox.parsers'
|
|
|
|
|
|
from typing import IO, Iterable
|
|
from time import mktime
|
|
from feedparser import parse as feedparser
|
|
|
|
from ..index.schema import Link
|
|
from ..util import (
|
|
htmldecode,
|
|
enforce_types
|
|
)
|
|
|
|
@enforce_types
|
|
def parse_generic_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
|
|
"""Parse RSS XML-format files into links"""
|
|
|
|
rss_file.seek(0)
|
|
feed = feedparser(rss_file.read())
|
|
for item in feed.entries:
|
|
url = item.link
|
|
title = item.title
|
|
time = mktime(item.updated_parsed)
|
|
|
|
try:
|
|
tags = ','.join(map(lambda tag: tag.term, item.tags))
|
|
except AttributeError:
|
|
tags = ''
|
|
|
|
if url is None:
|
|
# Yielding a Link with no URL will
|
|
# crash on a URL validation assertion
|
|
continue
|
|
|
|
yield Link(
|
|
url=htmldecode(url),
|
|
timestamp=str(time),
|
|
title=htmldecode(title) or None,
|
|
tags=tags,
|
|
sources=[rss_file.name],
|
|
)
|
|
|
|
|
|
KEY = 'rss'
|
|
NAME = 'Generic RSS'
|
|
PARSER = parse_generic_rss_export
|