mirror of
https://github.com/ArchiveBox/ArchiveBox
synced 2024-11-10 14:44:18 +00:00
remove derivable link info from links
This commit is contained in:
Nick Sweeting
parent
eb003f6a26
commit
09d79e55a0
2 changed files with 22 additions and 27 deletions
|
|
@ -8,27 +8,22 @@ For examples of supported files see examples/.
|
|||
|
||||
Parsed link schema: {
|
||||
'url': 'https://example.com/example/?abc=123&xyc=345#lmnop',
|
||||
'domain': 'example.com',
|
||||
'base_url': 'example.com/example/',
|
||||
'timestamp': '15442123124234',
|
||||
'tags': 'abc,def',
|
||||
'title': 'Example.com Page Title',
|
||||
'tags': 'abc,def',
|
||||
'sources': ['ril_export.html', 'downloads/getpocket.com.txt'],
|
||||
}
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
|
||||
from datetime import datetime
|
||||
from collections import OrderedDict
|
||||
import xml.etree.ElementTree as etree
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from config import ANSI, SHOW_PROGRESS
|
||||
from config import ANSI
|
||||
from util import (
|
||||
domain,
|
||||
base_url,
|
||||
str_between,
|
||||
get_link_type,
|
||||
URL_REGEX,
|
||||
|
|
@ -90,8 +85,6 @@ def parse_pocket_html_export(html_file):
|
|||
time = datetime.fromtimestamp(float(match.group(2)))
|
||||
info = {
|
||||
'url': fixed_url,
|
||||
'domain': domain(fixed_url),
|
||||
'base_url': base_url(fixed_url),
|
||||
'timestamp': str(time.timestamp()),
|
||||
'tags': match.group(3),
|
||||
'title': match.group(4).replace(' — Readability', '').replace('http://www.readability.com/read?url=', '') or None,
|
||||
|
|
@ -127,8 +120,6 @@ def parse_pinboard_json_export(json_file):
|
|||
title = erg['title'].strip()
|
||||
info = {
|
||||
'url': url,
|
||||
'domain': domain(url),
|
||||
'base_url': base_url(url),
|
||||
'timestamp': timestamp,
|
||||
'tags': erg.get('tags') or '',
|
||||
'title': title or None,
|
||||
|
|
@ -137,6 +128,7 @@ def parse_pinboard_json_export(json_file):
|
|||
info['type'] = get_link_type(info)
|
||||
yield info
|
||||
|
||||
|
||||
def parse_rss_export(rss_file):
|
||||
"""Parse RSS XML-format files into links"""
|
||||
|
||||
|
|
@ -166,8 +158,6 @@ def parse_rss_export(rss_file):
|
|||
|
||||
info = {
|
||||
'url': url,
|
||||
'domain': domain(url),
|
||||
'base_url': base_url(url),
|
||||
'timestamp': str(time.timestamp()),
|
||||
'tags': '',
|
||||
'title': title or None,
|
||||
|
|
@ -177,6 +167,7 @@ def parse_rss_export(rss_file):
|
|||
|
||||
yield info
|
||||
|
||||
|
||||
def parse_shaarli_rss_export(rss_file):
|
||||
"""Parse Shaarli-specific RSS XML-format files into links"""
|
||||
|
||||
|
|
@ -207,8 +198,6 @@ def parse_shaarli_rss_export(rss_file):
|
|||
|
||||
info = {
|
||||
'url': url,
|
||||
'domain': domain(url),
|
||||
'base_url': base_url(url),
|
||||
'timestamp': str(time.timestamp()),
|
||||
'tags': '',
|
||||
'title': title or None,
|
||||
|
|
@ -218,6 +207,7 @@ def parse_shaarli_rss_export(rss_file):
|
|||
|
||||
yield info
|
||||
|
||||
|
||||
def parse_netscape_html_export(html_file):
|
||||
"""Parse netscape-format bookmarks export files (produced by all browsers)"""
|
||||
|
||||
|
|
@ -234,8 +224,6 @@ def parse_netscape_html_export(html_file):
|
|||
|
||||
info = {
|
||||
'url': url,
|
||||
'domain': domain(url),
|
||||
'base_url': base_url(url),
|
||||
'timestamp': str(time.timestamp()),
|
||||
'tags': "",
|
||||
'title': match.group(3).strip() or None,
|
||||
|
|
@ -267,8 +255,6 @@ def parse_pinboard_rss_export(rss_file):
|
|||
time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
|
||||
info = {
|
||||
'url': url,
|
||||
'domain': domain(url),
|
||||
'base_url': base_url(url),
|
||||
'timestamp': str(time.timestamp()),
|
||||
'tags': tags,
|
||||
'title': title or None,
|
||||
|
|
@ -292,8 +278,6 @@ def parse_medium_rss_export(rss_file):
|
|||
time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %Z")
|
||||
info = {
|
||||
'url': url,
|
||||
'domain': domain(url),
|
||||
'base_url': base_url(url),
|
||||
'timestamp': str(time.timestamp()),
|
||||
'tags': '',
|
||||
'title': title or None,
|
||||
|
|
@ -316,8 +300,6 @@ def parse_plain_text_export(text_file):
|
|||
url = url.strip()
|
||||
info = {
|
||||
'url': url,
|
||||
'domain': domain(url),
|
||||
'base_url': base_url(url),
|
||||
'timestamp': str(datetime.now().timestamp()),
|
||||
'tags': '',
|
||||
'title': None,
|
||||
|
|
|
|||
|
|
@ -559,18 +559,30 @@ def wget_output_path(link, look_in=None):
|
|||
def derived_link_info(link):
|
||||
"""extend link info with the archive urls and other derived data"""
|
||||
|
||||
url = link['url']
|
||||
|
||||
link_info = {
|
||||
**link,
|
||||
'title': link['title'] or url,
|
||||
'date': datetime.fromtimestamp(Decimal(link['timestamp'])).strftime('%Y-%m-%d %H:%M'),
|
||||
'google_favicon_url': 'https://www.google.com/s2/favicons?domain={domain}'.format(**link),
|
||||
'base_url': base_url(url),
|
||||
'domain': domain(url),
|
||||
'basename': basename(url),
|
||||
'path': path(url),
|
||||
|
||||
# Archive Method Output URLs
|
||||
'favicon_url': 'archive/{timestamp}/favicon.ico'.format(**link),
|
||||
'google_favicon_url': 'https://www.google.com/s2/favicons?domain={domain}'.format(**link),
|
||||
'files_url': 'archive/{timestamp}/index.html'.format(**link),
|
||||
'archive_url': 'archive/{}/{}'.format(link['timestamp'], wget_output_path(link) or 'index.html'),
|
||||
'warc_url': 'archive/{timestamp}/warc'.format(**link),
|
||||
'pdf_link': 'archive/{timestamp}/output.pdf'.format(**link),
|
||||
'screenshot_link': 'archive/{timestamp}/screenshot.png'.format(**link),
|
||||
'dom_link': 'archive/{timestamp}/output.html'.format(**link),
|
||||
'archive_org_url': 'https://web.archive.org/web/{base_url}'.format(**link),
|
||||
'title': link['title'] or link['url'],
|
||||
'git_url': 'archive/{timestamp}/git'.format(**link),
|
||||
'media_url': 'archive/{timestamp}/media'.format(**link),
|
||||
|
||||
}
|
||||
|
||||
# PDF and images are handled slightly differently
|
||||
|
|
@ -583,6 +595,7 @@ def derived_link_info(link):
|
|||
'dom_link': 'archive/{timestamp}/{base_url}'.format(**link),
|
||||
'title': link['title'] or basename(link['url']),
|
||||
})
|
||||
|
||||
return link_info
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue