ArchiveBox/archivebox/links.py

155 lines
5.1 KiB
Python
Raw Normal View History

2017-10-19 00:08:33 +00:00
"""
In ArchiveBox, a Link represents a single entry that we track in the
2017-10-19 00:08:33 +00:00
json index. All links pass through all archiver functions and the latest,
2017-10-23 09:58:41 +00:00
most up-to-date canonical output for each is stored in "latest".
2017-10-19 00:08:33 +00:00
Link {
2017-10-23 09:58:41 +00:00
timestamp: str, (how we uniquely id links) _ _ _ _ ___
2017-10-19 00:08:33 +00:00
url: str, | \ / \ |\| ' |
base_url: str, |_/ \_/ | | |
domain: str, _ _ _ _ _ _
tags: str, |_) /| |\| | / `
type: str, | /"| | | | \_,
title: str, ,-'"`-.
sources: [str], /// / @ @ \ \\\\
2017-10-23 09:58:41 +00:00
latest: { \ :=| ,._,. |=: /
2017-10-19 00:08:33 +00:00
..., || ,\ \_../ /. ||
pdf: 'output.pdf', ||','`-._))'`.`||
wget: 'example.com/1234/index.html' `-' (/ `-'
},
history: {
...
pdf: [
{timestamp: 15444234325, status: 'skipped', result='output.pdf'},
...
],
wget: [
{timestamp: 11534435345, status: 'succeded', result='donuts.com/eat/them.html'}
]
},
}
"""
2018-04-17 07:22:59 +00:00
import datetime
2018-04-17 14:30:25 +00:00
from html import unescape
from collections import OrderedDict
2018-04-17 07:22:59 +00:00
2017-10-23 09:58:41 +00:00
from util import (
domain,
base_url,
str_between,
2017-10-23 09:58:41 +00:00
get_link_type,
merge_links,
2018-04-17 14:30:25 +00:00
wget_output_path,
2017-10-23 09:58:41 +00:00
)
2018-04-17 07:22:59 +00:00
from config import ANSI
2017-10-23 09:58:41 +00:00
def validate_links(links):
2017-10-23 09:58:41 +00:00
links = archivable_links(links) # remove chrome://, about:, mailto: etc.
2018-04-17 13:14:13 +00:00
links = uniquefied_links(links) # merge/dedupe duplicate timestamps & urls
2018-04-17 13:49:32 +00:00
links = sorted_links(links) # deterministically sort the links based on timstamp, url
if not links:
print('[X] No links found :(')
raise SystemExit(1)
2018-04-17 14:30:25 +00:00
for link in links:
link['title'] = unescape(link['title'])
link['latest'] = link.get('latest') or {}
latest = link['latest']
2018-04-17 14:30:25 +00:00
if not link['latest'].get('wget'):
link['latest']['wget'] = wget_output_path(link)
if not link['latest'].get('pdf'):
2018-04-17 21:16:29 +00:00
link['latest']['pdf'] = None
2018-04-17 14:30:25 +00:00
if not link['latest'].get('screenshot'):
2018-04-17 21:16:29 +00:00
link['latest']['screenshot'] = None
2018-04-17 14:30:25 +00:00
2018-06-10 22:45:41 +00:00
if not link['latest'].get('dom'):
link['latest']['dom'] = None
if not latest.get('favicon'):
latest['favicon'] = None
return list(links)
2018-10-21 20:36:32 +00:00
def new_links(all_links, existing_links):
"""
2018-10-21 20:36:32 +00:00
Return all links which are in the all_links but not in the existing_links.
This is used to determine which links are new and not indexed jet. Set the
ONLY_NEW environment variable to activate this filter mechanism.
"""
existing_urls = {link['url'] for link in existing_links}
return [link for link in all_links if link['url'] not in existing_urls]
2017-10-23 09:58:41 +00:00
def archivable_links(links):
"""remove chrome://, about:// or other schemed links that cant be archived"""
return (
link
for link in links
if any(link['url'].startswith(s) for s in ('http://', 'https://', 'ftp://'))
)
def uniquefied_links(sorted_links):
"""
ensures that all non-duplicate links have monotonically increasing timestamps
"""
unique_urls = OrderedDict()
lower = lambda url: url.lower().strip()
without_www = lambda url: url.replace('://www.', '://', 1)
without_trailing_slash = lambda url: url[:-1] if url[-1] == '/' else url.replace('/?', '?')
for link in sorted_links:
2017-10-19 00:33:31 +00:00
fuzzy_url = without_www(without_trailing_slash(lower(link['url'])))
if fuzzy_url in unique_urls:
# merge with any other links that share the same url
2017-10-19 00:33:31 +00:00
link = merge_links(unique_urls[fuzzy_url], link)
unique_urls[fuzzy_url] = link
unique_timestamps = OrderedDict()
2017-10-19 00:33:31 +00:00
for link in unique_urls.values():
link['timestamp'] = lowest_uniq_timestamp(unique_timestamps, link['timestamp'])
unique_timestamps[link['timestamp']] = link
return unique_timestamps.values()
2017-10-23 09:58:41 +00:00
def sorted_links(links):
sort_func = lambda link: (link['timestamp'].split('.', 1)[0], link['url'])
2017-10-23 09:58:41 +00:00
return sorted(links, key=sort_func, reverse=True)
def links_after_timestamp(links, timestamp=None):
if not timestamp:
yield from links
return
for link in links:
try:
if float(link['timestamp']) <= float(timestamp):
yield link
except (ValueError, TypeError):
print('Resume value and all timestamp values must be valid numbers.')
2017-10-19 00:33:31 +00:00
def lowest_uniq_timestamp(used_timestamps, timestamp):
"""resolve duplicate timestamps by appending a decimal 1234, 1234 -> 1234.1, 1234.2"""
2017-10-19 00:33:31 +00:00
timestamp = timestamp.split('.')[0]
nonce = 0
# first try 152323423 before 152323423.0
if timestamp not in used_timestamps:
return timestamp
new_timestamp = '{}.{}'.format(timestamp, nonce)
while new_timestamp in used_timestamps:
nonce += 1
new_timestamp = '{}.{}'.format(timestamp, nonce)
return new_timestamp