Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-10 06:34:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

major refactor + ability to handle http downloads

Browse source
This commit is contained in:
Nick Sweeting 2017-10-18 17:38:17 -05:00
parent f31da350c5
commit eb47155a12
12 changed files with 1325 additions and 659 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -2,6 +2,8 @@
pocket/
bookmarks/
pinboard/
html/
downloads/
# Byte-compiled / optimized / DLL files
__pycache__/

2
README.md
View file

@ -174,7 +174,7 @@ My published archive as an example: [sweeting.me/pocket](https://home.sweeting.m
If you don't like running random setup scripts off the internet (:+1:), you can follow these manual setup instructions.
**1. Install dependencies:** `chromium >= 59`,` wget >= 1.16`, `python3 >= 3.5` (google-chrome >= v59 also works well)
**1. Install dependencies:** `chromium >= 59`,` wget >= 1.16`, `python3 >= 3.5` (`google-chrome >= v59` works fine as well)
If you already have Google Chrome installed, or wish to use that instead of Chromium, follow the [Google Chrome Instructions](#google-chrome-instructions).

107
archive.py
View file

@ -3,74 +3,79 @@
# Nick Sweeting 2017 | MIT License
# https://github.com/pirate/bookmark-archiver
import os
import sys
from datetime import datetime
from links import validate_links
from parse import parse_export
from index import dump_index
from fetch import dump_website
from archive_methods import archive_links, _RESULTS_TOTALS
from index import (
write_links_index,
write_link_index,
parse_json_links_index,
parse_json_link_index,
)
from config import (
ARCHIVE_PERMISSIONS,
ARCHIVE_DIR,
HTML_FOLDER,
ARCHIVE_FOLDER,
ANSI,
TIMEOUT,
)
from util import (
download_url,
check_dependencies,
progress,
)
DESCRIPTION = 'Bookmark Archiver: Create a browsable html archive of a list of links.'
__DOCUMENTATION__ = 'https://github.com/pirate/bookmark-archiver'
def create_archive(export_file, service=None, resume=None):
def update_archive(export_path, resume=None, append=True):
"""update or create index.html and download archive of all links"""
print('[*] [{}] Starting archive from {} export file.'.format(
start_ts = datetime.now().timestamp()
# parse an validate the export file
new_links = validate_links(parse_export(export_path))
# load existing links if archive folder is present
if append:
existing_links = parse_json_links_index(HTML_FOLDER)
links = validate_links(existing_links + new_links)
else:
existing_links = []
# merge existing links and new links
num_new_links = len(links) - len(existing_links)
print('[*] [{}] Adding {} new links from {} to index'.format(
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
export_file,
num_new_links,
export_path,
))
with open(export_file, 'r', encoding='utf-8') as f:
links, service = parse_export(f, service=service)
# write link index html & json
write_links_index(HTML_FOLDER, links)
if resume:
try:
links = [
link
for link in links
if float(link['timestamp']) >= float(resume)
]
except TypeError:
print('Resume value and all timestamp values must be valid numbers.')
# loop over links and archive them
archive_links(ARCHIVE_FOLDER, links, export_path, resume=resume)
if not links or not service:
print('[X] No links found in {}, is it a {} export file?'.format(export_file, service))
raise SystemExit(1)
if not os.path.exists(os.path.join(ARCHIVE_DIR, service)):
os.makedirs(os.path.join(ARCHIVE_DIR, service))
if not os.path.exists(os.path.join(ARCHIVE_DIR, service, 'archive')):
os.makedirs(os.path.join(ARCHIVE_DIR, service, 'archive'))
dump_index(links, service)
check_dependencies()
try:
for link in links:
dump_website(link, service)
except (KeyboardInterrupt, SystemExit, Exception) as e:
print('{red}[X] Archive creation stopped.{reset}'.format(**ANSI))
print(' Continue where you left off by running:')
print(' ./archive.py {} {} {}'.format(
export_file,
service,
link['timestamp'],
))
if not isinstance(e, KeyboardInterrupt):
raise e
raise SystemExit(1)
print('{}[√] [{}] Archive update complete.{}'.format(ANSI['green'], datetime.now().strftime('%Y-%m-%d %H:%M:%S'), ANSI['reset']))
# print timing information & summary
end_ts = datetime.now().timestamp()
seconds = round(end_ts - start_ts, 1)
duration = '{} min'.format(seconds / 60) if seconds > 60 else '{} sec'.format(seconds)
print('{}[√] [{}] Archive update complete ({}){}'.format(
ANSI['green'],
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
duration,
ANSI['reset'],
))
print(' - {} skipped'.format(_RESULTS_TOTALS['skipped']))
print(' - {} updates'.format(_RESULTS_TOTALS['succeded']))
print(' - {} errors'.format(_RESULTS_TOTALS['failed']))
if __name__ == '__main__':
@ -85,8 +90,10 @@ if __name__ == '__main__':
print("")
raise SystemExit(0)
export_file = sys.argv[1] # path to export file
export_type = sys.argv[2] if argc > 2 else None # select export_type for file format select
resume_from = sys.argv[3] if argc > 3 else None # timestamp to resume dowloading from
export_path = sys.argv[1] # path to export file
resume_from = sys.argv[2] if argc > 2 else None # timestamp to resume dowloading from
create_archive(export_file, service=export_type, resume=resume_from)
if any(export_path.startswith(s) for s in ('http://', 'https://', 'ftp://')):
export_path = download_url(export_path)
update_archive(export_path, resume=resume_from)

410
archive_methods.py Normal file
View file

@ -0,0 +1,410 @@
import os
from functools import wraps
from datetime import datetime
from subprocess import run, PIPE, DEVNULL
from index import html_appended_url, parse_json_link_index, write_link_index
from links import links_after_timestamp
from config import (
ARCHIVE_PERMISSIONS,
ARCHIVE_DIR,
CHROME_BINARY,
FETCH_WGET,
FETCH_WGET_REQUISITES,
FETCH_PDF,
FETCH_SCREENSHOT,
RESOLUTION,
SUBMIT_ARCHIVE_DOT_ORG,
FETCH_AUDIO,
FETCH_VIDEO,
FETCH_FAVICON,
WGET_USER_AGENT,
TIMEOUT,
ANSI,
)
from util import (
check_dependencies,
progress,
chmod_file,
)
_RESULTS_TOTALS = {
'skipped': 0,
'succeded': 0,
'failed': 0,
}
def attach_result_to_link(method):
def decorator(fetch_func):
@wraps(fetch_func)
def timed_fetch_func(out_dir, link, overwrite=False, **kwargs):
# initialize methods and history json field on link
link['methods'] = link.get('methods') or {}
link['methods'][method] = link['methods'].get(method) or None
link['history'] = link.get('history') or {}
link['history'][method] = link['history'].get(method) or []
start_ts = datetime.now().timestamp()
# if a valid method output is already present, dont run the fetch function
if link['methods'][method] and not overwrite:
print(' √ Skipping: {}'.format(method))
result = None
else:
print(' - Fetching: {}'.format(method))
result = fetch_func(out_dir, link, **kwargs)
end_ts = datetime.now().timestamp()
duration = str(end_ts * 1000 - start_ts * 1000).split('.')[0]
# append a history item recording fail/success
history_entry = {
'timestamp': str(start_ts).split('.')[0],
}
if result is None:
history_entry['status'] = 'skipped'
elif isinstance(result.get('output'), Exception):
history_entry['status'] = 'failed'
history_entry['duration'] = duration
history_entry.update(result or {})
link['history'][method].append(history_entry)
else:
history_entry['status'] = 'succeded'
history_entry['duration'] = duration
history_entry.update(result or {})
link['history'][method].append(history_entry)
link['methods'][method] = result['output']
_RESULTS_TOTALS[history_entry['status']] += 1
return link
return timed_fetch_func
return decorator
@attach_result_to_link('wget')
def fetch_wget(out_dir, link, requisites=FETCH_WGET_REQUISITES, timeout=TIMEOUT):
"""download full site using wget"""
if os.path.exists(os.path.join(out_dir, link['domain'])):
return {'output': html_appended_url(link), 'status': 'skipped'}
CMD = [
*'wget --timestamping --adjust-extension --no-parent'.split(' '), # Docs: https://www.gnu.org/software/wget/manual/wget.html
*(('--page-requisites', '--convert-links') if FETCH_WGET_REQUISITES else ()),
*(('--user-agent="{}"'.format(WGET_USER_AGENT),) if WGET_USER_AGENT else ()),
link['url'],
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=out_dir, timeout=timeout + 1) # index.html
end()
output = html_appended_url(link)
if result.returncode > 0:
print(' got wget response code {}:'.format(result.returncode))
print('\n'.join(' ' + line for line in result.stderr.decode().rsplit('\n', 10)[-10:] if line.strip()))
# raise Exception('Failed to wget download')
chmod_file(link['domain'], cwd=out_dir)
except Exception as e:
end()
print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
output = e
return {
'cmd': CMD,
'output': output,
}
@attach_result_to_link('pdf')
def fetch_pdf(out_dir, link, timeout=TIMEOUT):
"""print PDF of site to file using chrome --headless"""
if link['type'] in ('PDF', 'image'):
return {'output': html_appended_url(link)}
if os.path.exists(os.path.join(out_dir, 'output.pdf')):
return {'output': 'output.pdf', 'status': 'skipped'}
CMD = [
CHROME_BINARY,
*'--headless --disable-gpu --print-to-pdf'.split(' '),
link['url']
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=DEVNULL, stderr=PIPE, cwd=out_dir, timeout=timeout + 1) # output.pdf
end()
if result.returncode:
print(' ', result.stderr.decode())
raise Exception('Failed to print PDF')
chmod_file('output.pdf', cwd=out_dir)
output = 'output.pdf'
except Exception as e:
end()
print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
output = e
return {
'cmd': CMD,
'output': output,
}
@attach_result_to_link('screenshot')
def fetch_screenshot(out_dir, link, timeout=TIMEOUT, resolution=RESOLUTION):
"""take screenshot of site using chrome --headless"""
if link['type'] in ('PDF', 'image'):
return {'output': html_appended_url(link)}
if os.path.exists(os.path.join(out_dir, 'screenshot.png')):
return {'output': 'screenshot.png', 'status': 'skipped'}
CMD = [
CHROME_BINARY,
*'--headless --disable-gpu --screenshot'.split(' '),
'--window-size={}'.format(resolution),
link['url']
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=DEVNULL, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # sreenshot.png
end()
if result.returncode:
print(' ', result.stderr.decode())
raise Exception('Failed to take screenshot')
chmod_file('screenshot.png', cwd=out_dir)
output = 'screenshot.png'
except Exception as e:
end()
print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
output = e
return {
'cmd': CMD,
'output': output,
}
@attach_result_to_link('archive_org')
def archive_dot_org(out_dir, link, timeout=TIMEOUT):
"""submit site to archive.org for archiving via their service, save returned archive url"""
path = os.path.join(out_dir, 'archive.org.txt')
if os.path.exists(path):
archive_org_url = open(path, 'r').read().strip()
return {'output': archive_org_url, 'status': 'skipped'}
submit_url = 'https://web.archive.org/save/{}'.format(link['url'].split('?', 1)[0])
success = False
CMD = ['curl', '-I', submit_url]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=PIPE, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # archive.org.txt
end()
# Parse archive.org response headers
headers = result.stdout.splitlines()
content_location = [h for h in headers if b'Content-Location: ' in h]
errors = [h for h in headers if b'X-Archive-Wayback-Runtime-Error: ' in h]
if content_location:
archive_path = content_location[0].split(b'Content-Location: ', 1)[-1].decode('utf-8')
saved_url = 'https://web.archive.org{}'.format(archive_path)
success = True
elif len(errors) == 1 and b'RobotAccessControlException' in errors[0]:
output = submit_url
# raise Exception('Archive.org denied by {}/robots.txt'.format(link['domain']))
elif errors:
raise Exception(', '.join(e.decode() for e in errors))
else:
raise Exception('Failed to find "Content-Location" URL header in Archive.org response.')
except Exception as e:
end()
print(' Visit url to see output:', ' '.join(CMD))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
output = e
if success:
with open(os.path.join(out_dir, 'archive.org.txt'), 'w', encoding='utf-8') as f:
f.write(saved_url)
chmod_file('archive.org.txt', cwd=out_dir)
output = saved_url
return {
'cmd': CMD,
'output': output,
}
@attach_result_to_link('favicon')
def fetch_favicon(out_dir, link, timeout=TIMEOUT):
"""download site favicon from google's favicon api"""
if os.path.exists(os.path.join(out_dir, 'favicon.ico')):
return {'output': 'favicon.ico', 'status': 'skipped'}
CMD = ['curl', 'https://www.google.com/s2/favicons?domain={domain}'.format(**link)]
fout = open('{}/favicon.ico'.format(out_dir), 'w')
end = progress(timeout, prefix=' ')
try:
run(CMD, stdout=fout, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # favicon.ico
fout.close()
end()
chmod_file('favicon.ico', cwd=out_dir)
output = 'favicon.ico'
except Exception as e:
fout.close()
end()
print(' Run to see full output:', ' '.join(CMD))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
output = e
return {
'cmd': CMD,
'output': output,
}
# @attach_result_to_link('audio')
# def fetch_audio(out_dir, link, timeout=TIMEOUT):
# """Download audio rip using youtube-dl"""
# if link['type'] not in ('soundcloud',)\
# and 'audio' not in link['tags']:
# return
# path = os.path.join(out_dir, 'audio')
# if not os.path.exists(path) or overwrite:
# print(' - Downloading audio')
# CMD = [
# "youtube-dl -x --audio-format mp3 --audio-quality 0 -o '%(title)s.%(ext)s'",
# link['url'],
# ]
# end = progress(timeout, prefix=' ')
# try:
# result = run(CMD, stdout=DEVNULL, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # audio/audio.mp3
# end()
# if result.returncode:
# print(' ', result.stderr.decode())
# raise Exception('Failed to download audio')
# chmod_file('audio.mp3', cwd=out_dir)
# return 'audio.mp3'
# except Exception as e:
# end()
# print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
# print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
# raise
# else:
# print(' √ Skipping audio download')
# @attach_result_to_link('video')
# def fetch_video(out_dir, link, timeout=TIMEOUT):
# """Download video rip using youtube-dl"""
# if link['type'] not in ('youtube', 'youku', 'vimeo')\
# and 'video' not in link['tags']:
# return
# path = os.path.join(out_dir, 'video')
# if not os.path.exists(path) or overwrite:
# print(' - Downloading video')
# CMD = [
# "youtube-dl -x --video-format mp4 --audio-quality 0 -o '%(title)s.%(ext)s'",
# link['url'],
# ]
# end = progress(timeout, prefix=' ')
# try:
# result = run(CMD, stdout=DEVNULL, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # video/movie.mp4
# end()
# if result.returncode:
# print(' ', result.stderr.decode())
# raise Exception('Failed to download video')
# chmod_file('video.mp4', cwd=out_dir)
# return 'video.mp4'
# except Exception as e:
# end()
# print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
# print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
# raise
# else:
# print(' √ Skipping video download')
def archive_links(out_dir, links, export_path, resume=None):
check_dependencies()
to_archive = links_after_timestamp(links, resume)
try:
for idx, link in enumerate(to_archive):
out_dir = os.path.join(out_dir, link['timestamp'])
archive_link(out_dir, link)
except (KeyboardInterrupt, SystemExit, Exception) as e:
print('{red}[X] Archive update stopped on #{idx} out of {total} links{reset}'.format(
**ANSI,
idx=idx,
total=len(list(to_archive)),
))
print(' Continue where you left off by running:')
print(' ./archive.py {} {}'.format(
export_path,
link['timestamp'],
))
if not isinstance(e, KeyboardInterrupt):
raise e
raise SystemExit(1)
def archive_link(out_dir, link, overwrite=False, permissions=ARCHIVE_PERMISSIONS):
"""download the DOM, PDF, and a screenshot into a folder named after the link's timestamp"""
link = {**parse_json_link_index(out_dir), **link}
log_link_archive(out_dir, link)
if FETCH_WGET:
link = fetch_wget(out_dir, link, overwrite=overwrite)
if FETCH_PDF:
link = fetch_pdf(out_dir, link, overwrite=overwrite)
if FETCH_SCREENSHOT:
link = fetch_screenshot(out_dir, link, overwrite=overwrite)
if SUBMIT_ARCHIVE_DOT_ORG:
link = archive_dot_org(out_dir, link, overwrite=overwrite)
# if FETCH_AUDIO:
# link = fetch_audio(out_dir, link, overwrite=overwrite)
# if FETCH_VIDEO:
# link = fetch_video(out_dir, link, overwrite=overwrite)
if FETCH_FAVICON:
link = fetch_favicon(out_dir, link, overwrite=overwrite)
write_link_index(out_dir, link)
return link
def log_link_archive(out_dir, link):
update_existing = os.path.exists(out_dir)
if not update_existing:
os.makedirs(out_dir)
run(['chmod', ARCHIVE_PERMISSIONS, out_dir], timeout=5)
print('[{symbol_color}{symbol}{reset}] [{timestamp}] "{title}": {blue}{base_url}{reset}'.format(
symbol='*' if update_existing else '+',
symbol_color=ANSI['black' if update_existing else 'green'],
**link,
**ANSI,
))
if link['type']:
print(' i Type: {}'.format(link['type']))

147
config.py
View file

@ -1,10 +1,8 @@
import os
import sys
import time
import shutil
from subprocess import run, PIPE, DEVNULL
from multiprocessing import Process
from subprocess import run, PIPE
# os.getenv('VARIABLE', 'DEFAULT') gets the value of environment
# variable "VARIABLE" and if it is not set, sets it to 'DEFAULT'
@ -12,8 +10,10 @@ from multiprocessing import Process
# for boolean values, check to see if the string is 'true', and
# if so, the python variable will be True
IS_TTY = sys.stdout.isatty()
# *******************************************************************************
# *** TO SET YOUR PREFERENCES, EDIT THE VALUES HERE, or use the 'env' command ***
IS_TTY = sys.stdout.isatty()
USE_COLOR = os.getenv('USE_COLOR', str(IS_TTY) ).lower() == 'true'
SHOW_PROGRESS = os.getenv('SHOW_PROGRESS', str(IS_TTY) ).lower() == 'true'
FETCH_WGET = os.getenv('FETCH_WGET', 'True' ).lower() == 'true'
@ -31,9 +31,12 @@ CHROME_BINARY = os.getenv('CHROME_BINARY', 'chromium-browser'
WGET_BINARY = os.getenv('WGET_BINARY', 'wget' )
WGET_USER_AGENT = os.getenv('WGET_USER_AGENT', None)
TIMEOUT = int(os.getenv('TIMEOUT', '60'))
LINK_INDEX_TEMPLATE = os.getenv('LINK_INDEX_TEMPLATE', 'templates/link_index.html')
INDEX_TEMPLATE = os.getenv('INDEX_TEMPLATE', 'templates/index.html')
INDEX_ROW_TEMPLATE = os.getenv('INDEX_ROW_TEMPLATE', 'templates/index_row.html')
# *******************************************************************************
TERM_WIDTH = shutil.get_terminal_size((100, 10)).columns
ANSI = {
'reset': '\033[00;00m',
@ -50,6 +53,17 @@ if not USE_COLOR:
# dont show colors if USE_COLOR is False
ANSI = {k: '' for k in ANSI.keys()}
ROOT_FOLDER = os.path.dirname(os.path.abspath(__file__))
HTML_FOLDER = os.path.join(ARCHIVE_DIR, 'html')
ARCHIVE_FOLDER = os.path.join(HTML_FOLDER, 'archive')
try:
GIT_SHA = run(["git", "rev-list", "-1", "HEAD", "./"], stdout=PIPE, cwd=ROOT_FOLDER).stdout.strip().decode()
except Exception:
GIT_SHA = None
print('[!] Warning, you need git installed for some archiving features to save correct version numbers!')
if sys.stdout.encoding.upper() != 'UTF-8':
print('[X] Your system is running python3 scripts with a bad locale setting: {} (it should be UTF-8).'.format(sys.stdout.encoding))
print(' To fix it, add the line "export PYTHONIOENCODING=utf8" to your ~/.bashrc file (without quotes)')
@ -59,128 +73,3 @@ if sys.stdout.encoding.upper() != 'UTF-8':
print('')
print(' Alternatively, run this script with:')
print(' env PYTHONIOENCODING=utf8 ./archive.py export.html')
### Util Functions
def check_dependencies():
"""Check that all necessary dependencies are installed, and have valid versions"""
print('[*] Checking Dependencies:')
python_vers = float('{}.{}'.format(sys.version_info.major, sys.version_info.minor))
if python_vers < 3.5:
print('{}[X] Python version is not new enough: {} (>3.5 is required){}'.format(ANSI['red'], python_vers, ANSI['reset']))
print(' See https://github.com/pirate/bookmark-archiver#troubleshooting for help upgrading your Python installation.')
raise SystemExit(1)
if FETCH_PDF or FETCH_SCREENSHOT:
if run(['which', CHROME_BINARY]).returncode:
print('{}[X] Missing dependency: {}{}'.format(ANSI['red'], CHROME_BINARY, ANSI['reset']))
print(' Run ./setup.sh, then confirm it was installed with: {} --version'.format(CHROME_BINARY))
print(' See https://github.com/pirate/bookmark-archiver for help.')
raise SystemExit(1)
# parse chrome --version e.g. Google Chrome 61.0.3114.0 canary / Chromium 59.0.3029.110 built on Ubuntu, running on Ubuntu 16.04
try:
result = run([CHROME_BINARY, '--version'], stdout=PIPE)
version = result.stdout.decode('utf-8').replace('Google Chrome ', '').replace('Chromium ', '').split(' ', 1)[0].split('.', 1)[0] # TODO: regex might be better
if int(version) < 59:
print('{red}[X] Chrome version must be 59 or greater for headless PDF and screenshot saving{reset}'.format(**ANSI))
print(' See https://github.com/pirate/bookmark-archiver for help.')
raise SystemExit(1)
except (TypeError, OSError):
print('{red}[X] Failed to parse Chrome version, is it installed properly?{reset}'.format(**ANSI))
print(' Run ./setup.sh, then confirm it was installed with: {} --version'.format(CHROME_BINARY))
print(' See https://github.com/pirate/bookmark-archiver for help.')
raise SystemExit(1)
if FETCH_WGET:
if run(['which', 'wget']).returncode or run(['wget', '--version'], stdout=DEVNULL).returncode:
print('{red}[X] Missing dependency: wget{reset}'.format(**ANSI))
print(' Run ./setup.sh, then confirm it was installed with: {} --version'.format('wget'))
print(' See https://github.com/pirate/bookmark-archiver for help.')
raise SystemExit(1)
if FETCH_FAVICON or SUBMIT_ARCHIVE_DOT_ORG:
if run(['which', 'curl']).returncode or run(['curl', '--version'], stdout=DEVNULL).returncode:
print('{red}[X] Missing dependency: curl{reset}'.format(**ANSI))
print(' Run ./setup.sh, then confirm it was installed with: {} --version'.format('curl'))
print(' See https://github.com/pirate/bookmark-archiver for help.')
raise SystemExit(1)
if FETCH_AUDIO or FETCH_VIDEO:
if run(['which', 'youtube-dl']).returncode or run(['youtube-dl', '--version'], stdout=DEVNULL).returncode:
print('{red}[X] Missing dependency: youtube-dl{reset}'.format(**ANSI))
print(' Run ./setup.sh, then confirm it was installed with: {} --version'.format('youtube-dl'))
print(' See https://github.com/pirate/bookmark-archiver for help.')
raise SystemExit(1)
def chmod_file(path, cwd='.', permissions=ARCHIVE_PERMISSIONS, timeout=30):
"""chmod -R <permissions> <cwd>/<path>"""
if not os.path.exists(os.path.join(cwd, path)):
raise Exception('Failed to chmod: {} does not exist (did the previous step fail?)'.format(path))
chmod_result = run(['chmod', '-R', permissions, path], cwd=cwd, stdout=DEVNULL, stderr=PIPE, timeout=timeout)
if chmod_result.returncode == 1:
print(' ', chmod_result.stderr.decode())
raise Exception('Failed to chmod {}/{}'.format(cwd, path))
def progress(seconds=TIMEOUT, prefix=''):
"""Show a (subprocess-controlled) progress bar with a <seconds> timeout,
returns end() function to instantly finish the progress
"""
if not SHOW_PROGRESS:
return lambda: None
chunk = '█' if sys.stdout.encoding == 'UTF-8' else '#'
chunks = TERM_WIDTH - len(prefix) - 20 # number of progress chunks to show (aka max bar width)
def progress_bar(seconds=seconds, prefix=prefix):
"""show timer in the form of progress bar, with percentage and seconds remaining"""
try:
for s in range(seconds * chunks):
progress = s / chunks / seconds * 100
bar_width = round(progress/(100/chunks))
# ████████████████████ 0.9% (1/60sec)
sys.stdout.write('\r{0}{1}{2}{3} {4}% ({5}/{6}sec)'.format(
prefix,
ANSI['green'],
(chunk * bar_width).ljust(chunks),
ANSI['reset'],
round(progress, 1),
round(s/chunks),
seconds,
))
sys.stdout.flush()
time.sleep(1 / chunks)
# ██████████████████████████████████ 100.0% (60/60sec)
sys.stdout.write('\r{0}{1}{2}{3} {4}% ({5}/{6}sec)\n'.format(
prefix,
ANSI['red'],
chunk * chunks,
ANSI['reset'],
100.0,
seconds,
seconds,
))
sys.stdout.flush()
except KeyboardInterrupt:
print()
pass
p = Process(target=progress_bar)
p.start()
def end():
"""immediately finish progress and clear the progressbar line"""
p.terminate()
sys.stdout.write('\r{}{}\r'.format((' ' * TERM_WIDTH), ANSI['reset'])) # clear whole terminal line
sys.stdout.flush()
return end

299
fetch.py
View file

@ -1,299 +0,0 @@
import os
import json
from datetime import datetime
from subprocess import run, PIPE, DEVNULL
from parse import derived_link_info
from config import (
ARCHIVE_PERMISSIONS,
ARCHIVE_DIR,
CHROME_BINARY,
FETCH_WGET,
FETCH_WGET_REQUISITES,
FETCH_PDF,
FETCH_SCREENSHOT,
RESOLUTION,
SUBMIT_ARCHIVE_DOT_ORG,
FETCH_AUDIO,
FETCH_VIDEO,
FETCH_FAVICON,
WGET_USER_AGENT,
TIMEOUT,
ANSI,
progress,
chmod_file,
)
def fetch_wget(out_dir, link, overwrite=False, requisites=True, timeout=TIMEOUT):
"""download full site using wget"""
if not os.path.exists(os.path.join(out_dir, link['domain'])) or overwrite:
print(' - Downloading full site')
CMD = [
*'wget --timestamping --adjust-extension --no-parent'.split(' '), # Docs: https://www.gnu.org/software/wget/manual/wget.html
*(('--page-requisites', '--convert-links') if requisites else ()),
*(('--user-agent="{}"'.format(WGET_USER_AGENT),) if WGET_USER_AGENT else ()),
link['url'],
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=out_dir, timeout=timeout + 1) # index.html
end()
if result.returncode > 0:
print(' wget output:')
print('\n'.join(' ' + line for line in result.stderr.decode().rsplit('\n', 10)[-10:] if line.strip()))
raise Exception('Failed to wget download')
chmod_file(link['domain'], cwd=out_dir)
except Exception as e:
end()
print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
else:
print(' √ Skipping site download')
def fetch_pdf(out_dir, link, overwrite=False, timeout=TIMEOUT, chrome_binary=CHROME_BINARY):
"""print PDF of site to file using chrome --headless"""
path = os.path.join(out_dir, 'output.pdf')
if (not os.path.exists(path) or overwrite) and link['type'] not in ('PDF', 'image'):
print(' - Printing PDF')
CMD = [
chrome_binary,
*'--headless --disable-gpu --print-to-pdf'.split(' '),
link['url']
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=DEVNULL, stderr=PIPE, cwd=out_dir, timeout=timeout + 1) # output.pdf
end()
if result.returncode:
print(' ', result.stderr.decode())
raise Exception('Failed to print PDF')
chmod_file('output.pdf', cwd=out_dir)
except Exception as e:
end()
print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
else:
print(' √ Skipping PDF print')
def fetch_screenshot(out_dir, link, overwrite=False, timeout=TIMEOUT, chrome_binary=CHROME_BINARY, resolution=RESOLUTION):
"""take screenshot of site using chrome --headless"""
path = os.path.join(out_dir, 'screenshot.png')
if (not os.path.exists(path) or overwrite) and link['type'] not in ('PDF', 'image'):
print(' - Snapping Screenshot')
CMD = [
chrome_binary,
*'--headless --disable-gpu --screenshot'.split(' '),
'--window-size={}'.format(resolution),
link['url']
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=DEVNULL, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # sreenshot.png
end()
if result.returncode:
print(' ', result.stderr.decode())
raise Exception('Failed to take screenshot')
chmod_file('screenshot.png', cwd=out_dir)
except Exception as e:
end()
print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
else:
print(' √ Skipping screenshot')
def archive_dot_org(out_dir, link, overwrite=False, timeout=TIMEOUT):
"""submit site to archive.org for archiving via their service, save returned archive url"""
path = os.path.join(out_dir, 'archive.org.txt')
if not os.path.exists(path) or overwrite:
print(' - Submitting to archive.org')
submit_url = 'https://web.archive.org/save/{}'.format(link['url'].split('?', 1)[0])
success = False
CMD = ['curl', '-I', submit_url]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=PIPE, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # archive.org.txt
end()
# Parse archive.org response headers
headers = result.stdout.splitlines()
content_location = [h for h in headers if b'Content-Location: ' in h]
errors = [h for h in headers if b'X-Archive-Wayback-Runtime-Error: ' in h]
if content_location:
archive_path = content_location[0].split(b'Content-Location: ', 1)[-1].decode('utf-8')
saved_url = 'https://web.archive.org{}'.format(archive_path)
success = True
elif len(errors) == 1 and b'RobotAccessControlException' in errors[0]:
raise Exception('Archive.org denied by {}/robots.txt'.format(link['domain']))
elif errors:
raise Exception(', '.join(e.decode() for e in errors))
else:
raise Exception('Failed to find "Content-Location" URL header in Archive.org response.')
except Exception as e:
end()
print(' Visit url to see output:', ' '.join(CMD))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
if success:
with open(os.path.join(out_dir, 'archive.org.txt'), 'w', encoding='utf-8') as f:
f.write(saved_url)
chmod_file('archive.org.txt', cwd=out_dir)
else:
print(' √ Skipping archive.org')
def fetch_favicon(out_dir, link, overwrite=False, timeout=TIMEOUT):
"""download site favicon from google's favicon api"""
path = os.path.join(out_dir, 'favicon.ico')
if not os.path.exists(path) or overwrite:
print(' - Fetching Favicon')
CMD = ['curl', 'https://www.google.com/s2/favicons?domain={domain}'.format(**link)]
fout = open('{}/favicon.ico'.format(out_dir), 'w')
end = progress(timeout, prefix=' ')
try:
run(CMD, stdout=fout, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # favicon.ico
end()
chmod_file('favicon.ico', cwd=out_dir)
except Exception as e:
end()
print(' Run to see full output:', ' '.join(CMD))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
fout.close()
else:
print(' √ Skipping favicon')
def fetch_audio(out_dir, link, overwrite=False, timeout=TIMEOUT):
"""Download audio rip using youtube-dl"""
if link['type'] not in ('soundcloud',):
return
path = os.path.join(out_dir, 'audio')
if not os.path.exists(path) or overwrite:
print(' - Downloading audio')
CMD = [
"youtube-dl -x --audio-format mp3 --audio-quality 0 -o '%(title)s.%(ext)s'",
link['url'],
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=DEVNULL, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # audio/audio.mp3
end()
if result.returncode:
print(' ', result.stderr.decode())
raise Exception('Failed to download audio')
chmod_file('audio', cwd=out_dir)
except Exception as e:
end()
print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
else:
print(' √ Skipping audio download')
def fetch_video(out_dir, link, overwrite=False, timeout=TIMEOUT):
"""Download video rip using youtube-dl"""
if link['type'] not in ('youtube', 'youku', 'vimeo'):
return
path = os.path.join(out_dir, 'video')
if not os.path.exists(path) or overwrite:
print(' - Downloading video')
CMD = [
"youtube-dl -x --video-format mp4 --audio-quality 0 -o '%(title)s.%(ext)s'",
link['url'],
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=DEVNULL, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # video/movie.mp4
end()
if result.returncode:
print(' ', result.stderr.decode())
raise Exception('Failed to download video')
chmod_file('video', cwd=out_dir)
except Exception as e:
end()
print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
else:
print(' √ Skipping video download')
def dump_link_info(out_dir, link, overwrite=False):
"""write a json file with some info about the link"""
info_file_path = os.path.join(out_dir, 'link.json')
if (not os.path.exists(info_file_path) or overwrite):
print(' - Creating link info file')
try:
link_json = derived_link_info(link)
link_json['archived_timstamp'] = str(datetime.now().timestamp()).split('.')[0]
with open(info_file_path, 'w', encoding='utf-8') as link_file:
link_file.write(json.dumps(
link_json,
indent=4,
default=str) + '\n')
chmod_file('link.json', cwd=out_dir)
except Exception as e:
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
else:
print(' √ Skipping link info file')
def dump_website(link, service, overwrite=False, permissions=ARCHIVE_PERMISSIONS):
"""download the DOM, PDF, and a screenshot into a folder named after the link's timestamp"""
print('[{green}+{reset}] [{timestamp} ({time})] "{title}": {blue}{base_url}{reset}'.format(**link, **ANSI))
out_dir = os.path.join(ARCHIVE_DIR, service, 'archive', link['timestamp'])
if not os.path.exists(out_dir):
os.makedirs(out_dir)
run(['chmod', permissions, out_dir], timeout=5)
if link['type']:
print(' i Type: {}'.format(link['type']))
if not (link['url'].startswith('http') or link['url'].startswith('ftp')):
print(' {}X Skipping: invalid link.{}', ANSI['red'], ANSI['yellow'])
return
if FETCH_WGET:
fetch_wget(out_dir, link, overwrite=overwrite, requisites=FETCH_WGET_REQUISITES)
if FETCH_PDF:
fetch_pdf(out_dir, link, overwrite=overwrite, chrome_binary=CHROME_BINARY)
if FETCH_SCREENSHOT:
fetch_screenshot(out_dir, link, overwrite=overwrite, chrome_binary=CHROME_BINARY, resolution=RESOLUTION)
if SUBMIT_ARCHIVE_DOT_ORG:
archive_dot_org(out_dir, link, overwrite=overwrite)
if FETCH_AUDIO:
fetch_audio(out_dir, link, overwrite=overwrite)
if FETCH_VIDEO:
fetch_video(out_dir, link, overwrite=overwrite)
if FETCH_FAVICON:
fetch_favicon(out_dir, link, overwrite=overwrite)
dump_link_info(out_dir, link, overwrite=overwrite)

183
index.py
View file

@ -1,47 +1,196 @@
import os
import re
import json
from datetime import datetime
from string import Template
from parse import derived_link_info
from config import (
INDEX_TEMPLATE,
INDEX_ROW_TEMPLATE,
LINK_INDEX_TEMPLATE,
ARCHIVE_PERMISSIONS,
ARCHIVE_DIR,
ANSI,
chmod_file,
GIT_SHA,
)
from util import chmod_file
def dump_index(links, service):
"""create index.html file for a given list of links and service"""
### Homepage index for all the links
def parse_json_links_index(out_dir):
"""load the index in a given directory and merge it with the given link"""
index_path = os.path.join(out_dir, 'index.json')
if os.path.exists(index_path):
with open(index_path, 'r', encoding='utf-8') as f:
return json.load(f)['links']
return []
def write_links_index(out_dir, links):
"""create index.html file for a given list of links"""
if not os.path.exists(out_dir):
os.makedirs(out_dir)
print('[i] [{}] Updating {}{}{} links in archive index...'.format(
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
ANSI['green'],
len(links),
ANSI['reset'],
))
write_json_links_index(out_dir, links)
write_html_links_index(out_dir, links)
chmod_file(out_dir, permissions=ARCHIVE_PERMISSIONS)
def write_json_links_index(out_dir, links):
"""write the json link index to a given path"""
path = os.path.join(out_dir, 'index.json')
index_json = {
'info': 'Bookmark Archiver Index',
'help': 'https://github.com/pirate/bookmark-archiver',
'version': GIT_SHA,
'num_links': len(links),
'updated': str(datetime.now().timestamp()),
'links': links,
}
with open(path, 'w', encoding='utf-8') as f:
json.dump(index_json, f, indent=4, default=str)
chmod_file(path)
def write_html_links_index(out_dir, links):
"""write the html link index to a given path"""
path = os.path.join(out_dir, 'index.html')
with open(INDEX_TEMPLATE, 'r', encoding='utf-8') as f:
index_html = f.read()
# TODO: refactor this out into index_template.html
with open(INDEX_ROW_TEMPLATE, 'r', encoding='utf-8') as f:
link_html = f.read()
link_row_html = f.read()
article_rows = '\n'.join(
Template(link_html).substitute(**derived_link_info(link)) for link in links
link_rows = '\n'.join(
Template(link_row_html).substitute(**derived_link_info(link))
for link in links
)
template_vars = {
'num_links': len(links),
'date_updated': datetime.now().strftime('%Y-%m-%d'),
'time_updated': datetime.now().strftime('%Y-%m-%d %H:%M'),
'rows': article_rows,
'rows': link_rows,
}
with open(os.path.join(ARCHIVE_DIR, service, 'index.html'), 'w', encoding='utf-8') as f:
with open(path, 'w', encoding='utf-8') as f:
f.write(Template(index_html).substitute(**template_vars))
chmod_file(os.path.join(ARCHIVE_DIR, service), permissions=ARCHIVE_PERMISSIONS)
print('[+] [{}] Created archive index with {}{}{} links.'.format(
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
ANSI['green'],
len(links),
ANSI['reset'],
))
### Individual link index
def parse_json_link_index(out_dir):
"""load the index in a given directory and merge it with the given link"""
existing_index = os.path.join(out_dir, 'index.json')
if os.path.exists(existing_index):
with open(existing_index, 'r', encoding='utf-8') as f:
return json.load(f)
return {}
def write_link_index(out_dir, link):
link['updated'] = str(datetime.now().timestamp())
write_json_link_index(out_dir, link)
write_html_link_index(out_dir, link)
def write_json_link_index(out_dir, link):
"""write a json file with some info about the link"""
path = os.path.join(out_dir, 'index.json')
with open(path, 'w', encoding='utf-8') as f:
json.dump(link, f, indent=4, default=str)
chmod_file(path)
def write_html_link_index(out_dir, link):
with open(LINK_INDEX_TEMPLATE, 'r', encoding='utf-8') as f:
link_html = f.read()
path = os.path.join(out_dir, 'index.html')
with open(path, 'w', encoding='utf-8') as f:
f.write(Template(link_html).substitute({
**link,
**link['methods'],
'type': link['type'] or 'website',
'tags': link['tags'] or '',
'bookmarked': datetime.fromtimestamp(float(link['timestamp'])).strftime('%Y-%m-%d %H:%M'),
'updated': datetime.fromtimestamp(float(link['updated'])).strftime('%Y-%m-%d %H:%M'),
'archive_org': link['methods']['archive_org'] or 'https://web.archive.org/save/{}'.format(link['url']),
'wget': link['methods']['wget'] or link['domain'],
}))
chmod_file(path)
def html_appended_url(link):
"""calculate the path to the wgetted .html file, since wget may
adjust some paths to be different than the base_url path.
See docs on wget --adjust-extension."""
if link['type'] in ('PDF', 'image'):
return link['base_url']
split_url = link['url'].split('#', 1)
query = ('%3F' + link['url'].split('?', 1)[-1]) if '?' in link['url'] else ''
if re.search(".+\\.[Hh][Tt][Mm][Ll]?$", split_url[0], re.I | re.M):
# already ends in .html
return link['base_url']
else:
# .html needs to be appended
without_scheme = split_url[0].split('://', 1)[-1].split('?', 1)[0]
if without_scheme.endswith('/'):
if query:
return '#'.join([without_scheme + 'index.html' + query + '.html', *split_url[1:]])
return '#'.join([without_scheme + 'index.html', *split_url[1:]])
else:
if query:
return '#'.join([without_scheme + '/index.html' + query + '.html', *split_url[1:]])
elif '/' in without_scheme:
return '#'.join([without_scheme + '.html', *split_url[1:]])
return link['base_url'] + '/index.html'
def derived_link_info(link):
"""extend link info with the archive urls and other derived data"""
link_info = {
**link,
'date': datetime.fromtimestamp(float(link['timestamp'])).strftime('%Y-%m-%d %H:%M'),
'google_favicon_url': 'https://www.google.com/s2/favicons?domain={domain}'.format(**link),
'favicon_url': 'archive/{timestamp}/favicon.ico'.format(**link),
'files_url': 'archive/{timestamp}/'.format(**link),
'archive_url': 'archive/{}/{}'.format(link['timestamp'], html_appended_url(link)),
'pdf_link': 'archive/{timestamp}/output.pdf'.format(**link),
'screenshot_link': 'archive/{timestamp}/screenshot.png'.format(**link),
'archive_org_url': 'https://web.archive.org/web/{base_url}'.format(**link),
}
# PDF and images are handled slightly differently
# wget, screenshot, & pdf urls all point to the same file
if link['type'] in ('PDF', 'image'):
link_info.update({
'archive_url': 'archive/{timestamp}/{base_url}'.format(**link),
'pdf_link': 'archive/{timestamp}/{base_url}'.format(**link),
'screenshot_link': 'archive/{timestamp}/{base_url}'.format(**link),
'title': '{title} ({type})'.format(**link),
})
return link_info

112
links.py Normal file
View file

@ -0,0 +1,112 @@
from util import (
domain,
base_url,
get_str_between,
get_link_type,
)
def validate_links(links):
links = valid_links(links) # remove chrome://, about:, mailto: etc.
links = uniquefied_links(links) # fix duplicate timestamps, returns sorted list
links = sorted_links(links) # deterministically sort the links
if not links:
print('[X] No links found :(')
raise SystemExit(1)
return list(links)
def sorted_links(links):
return sorted(
links,
key=lambda link: (link['timestamp'], link['url']),
reverse=True,
)
def merge_links(link1, link2):
longer = lambda a, b, key: a[key] if len(a[key]) > len(b[key]) else b[key]
earlier = lambda a, b, key: a[key] if a[key] < b[key] else b[key]
url = longer(link1, link2, 'url')
earliest_ts = earlier(link1, link2, 'timestamp')
longest_title = longer(link1, link2, 'title')
cleanest_title = link1['title'] if '://' not in link1['title'] else link2['title']
link = {
'url': url,
'domain': domain(url),
'base_url': base_url(url),
'timestamp': earliest_ts,
'tags': longer(link1, link2, 'tags'),
'title': longest_title if '://' not in longest_title else cleanest_title,
'sources': list(set(link1['sources'] + link2['sources'])),
}
link['type'] = get_link_type(link)
return link
def uniquefied_links(sorted_links):
"""
ensures that all non-duplicate links have monotonically increasing timestamps
"""
seen_urls = {}
seen_timestamps = set()
lower = lambda url: url.lower().strip()
without_www = lambda url: url.replace('://www.', '://', 1)
without_trailing_slash = lambda url: url[:-1] if url[-1] == '/' else url.replace('/?', '?')
for link in sorted_links:
url = without_www(without_trailing_slash(lower(link['url'])))
if url in seen_urls:
# merge with any other links that share the same url
link = merge_links(seen_urls[url], link)
elif link['timestamp'] in seen_timestamps:
# add with incremented timestamp if earlier link exist with same timestamp
link['timestamp'] = next_uniq_timestamp(seen_timestamps, link['timestamp'])
seen_urls[url] = link
seen_timestamps.add(link['timestamp'])
return seen_urls.values()
def valid_links(links):
"""remove chrome://, about:// or other schemed links that cant be archived"""
return (
link
for link in links
if any(link['url'].startswith(s) for s in ('http://', 'https://', 'ftp://'))
)
def links_after_timestamp(links, timestamp=None):
if not timestamp:
yield from links
return
print('[.] [{}] Resuming...'.format(timestamp))
for link in links:
try:
if float(link['timestamp']) <= float(timestamp):
yield link
except (ValueError, TypeError):
print('Resume value and all timestamp values must be valid numbers.')
def next_uniq_timestamp(used_timestamps, timestamp):
"""resolve duplicate timestamps by appending a decimal 1234, 1234 -> 1234.1, 1234.2"""
if timestamp not in used_timestamps:
return timestamp
if '.' in timestamp:
timestamp, nonce = timestamp.split('.')
nonce = int(nonce)
else:
nonce = 1
new_timestamp = '{}.{}'.format(timestamp, nonce)
while new_timestamp in used_timestamps:
nonce += 1
new_timestamp = '{}.{}'.format(timestamp, nonce)
return new_timestamp

244
parse.py
View file

@ -1,56 +1,38 @@
import re
import time
import json
from datetime import datetime
from util import (
domain,
base_url,
get_str_between,
get_link_type,
)
def parse_export(file, service=None):
def parse_export(path):
"""parse a list of links dictionaries from a bookmark export file"""
links = []
with open(path, 'r', encoding='utf-8') as file:
for service, parser_func in get_parsers().items():
# otherwise try all parsers until one works
try:
links += list(parser_func(file))
if links:
break
except Exception as e:
pass
# if specific service was passed via command line
if service == "pocket":
links = parse_pocket_export(file)
elif service == "pinboard":
links = parse_json_export(file)
elif service == "bookmarks":
links = parse_bookmarks_export(file)
else:
# otherwise try all parsers until one works
try:
links = list(parse_json_export(file))
service = 'pinboard'
except Exception:
links = list(parse_pocket_export(file))
if links:
service = 'pocket'
else:
links = list(parse_bookmarks_export(file))
service = 'bookmarks'
return links
links = valid_links(links) # remove chrome://, about:, mailto: etc.
links = uniquefied_links(links) # fix duplicate timestamps, returns sorted list
return links, service
def get_link_type(link):
"""Certain types of links need to be handled specially, this figures out when that's the case"""
if link['base_url'].endswith('.pdf'):
return 'PDF'
elif link['base_url'].rsplit('.', 1) in ('pdf', 'png', 'jpg', 'jpeg', 'svg', 'bmp', 'gif', 'tiff', 'webp'):
return 'image'
elif 'wikipedia.org' in link['domain']:
return 'wiki'
elif 'youtube.com' in link['domain']:
return 'youtube'
elif 'soundcloud.com' in link['domain']:
return 'soundcloud'
elif 'youku.com' in link['domain']:
return 'youku'
elif 'vimeo.com' in link['domain']:
return 'vimeo'
return None
def get_parsers():
return {
'pocket': parse_pocket_export,
'pinboard': parse_json_export,
'bookmarks': parse_bookmarks_export,
'rss': parse_rss_export,
}
def parse_pocket_export(html_file):
"""Parse Pocket-format bookmarks export files (produced by getpocket.com/export/)"""
@ -61,15 +43,15 @@ def parse_pocket_export(html_file):
match = pattern.search(line)
if match:
fixed_url = match.group(1).replace('http://www.readability.com/read?url=', '') # remove old readability prefixes to get original url
without_scheme = fixed_url.replace('http://', '').replace('https://', '')
time = datetime.fromtimestamp(float(match.group(2)))
info = {
'url': fixed_url,
'domain': without_scheme.split('/', 1)[0], # without pathname
'base_url': without_scheme.split('?', 1)[0], # without query args
'time': datetime.fromtimestamp(int(match.group(2))).strftime('%Y-%m-%d %H:%M'),
'timestamp': match.group(2),
'domain': domain(fixed_url),
'base_url': base_url(fixed_url),
'timestamp': str(time.timestamp()),
'tags': match.group(3),
'title': match.group(4).replace(' — Readability', '').replace('http://www.readability.com/read?url=', '') or without_scheme,
'title': match.group(4).replace(' — Readability', '').replace('http://www.readability.com/read?url=', '') or base_url(fixed_url),
'sources': [html_file.name],
}
info['type'] = get_link_type(info)
yield info
@ -82,18 +64,59 @@ def parse_json_export(json_file):
for line in json_content:
if line:
erg = line
time = datetime.strptime(erg['time'].split(',', 1)[0], '%Y-%m-%dT%H:%M:%SZ')
info = {
'url': erg['href'],
'domain': erg['href'].replace('http://', '').replace('https://', '').split('/', 1)[0],
'base_url': erg['href'].replace('https://', '').replace('http://', '').split('?', 1)[0],
'time': datetime.fromtimestamp(int(time.mktime(time.strptime(erg['time'].split(',', 1)[0], '%Y-%m-%dT%H:%M:%SZ')))),
'timestamp': str(int(time.mktime(time.strptime(erg['time'].split(',', 1)[0], '%Y-%m-%dT%H:%M:%SZ')))),
'domain': domain(erg['href']),
'base_url': base_url(erg['href']),
'timestamp': str(time.timestamp()),
'tags': erg['tags'],
'title': erg['description'].replace(' — Readability', ''),
'sources': [json_file.name],
}
info['type'] = get_link_type(info)
yield info
def parse_rss_export(rss_file):
"""Parse RSS XML-format files into links"""
rss_file.seek(0)
items = rss_file.read().split('</item>\n<item>')
for item in items:
# example item:
# <item>
# <title><![CDATA[How JavaScript works: inside the V8 engine]]></title>
# <category>Unread</category>
# <link>https://blog.sessionstack.com/how-javascript-works-inside</link>
# <guid>https://blog.sessionstack.com/how-javascript-works-inside</guid>
# <pubDate>Mon, 21 Aug 2017 14:21:58 -0500</pubDate>
# </item>
trailing_removed = item.split('</item>', 1)[0]
leading_removed = trailing_removed.split('<item>', 1)[-1]
rows = leading_removed.split('\n')
row = lambda key: [r for r in rows if r.startswith('<{}>'.format(key))][0]
title = get_str_between(row('title'), '<![CDATA[', ']]')
url = get_str_between(row('link'), '<link>', '</link>')
ts_str = get_str_between(row('pubDate'), '<pubDate>', '</pubDate>')
time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %z")
info = {
'url': url,
'domain': domain(url),
'base_url': base_url(url),
'timestamp': str(time.timestamp()),
'tags': '',
'title': title,
'sources': [rss_file.name],
}
info['type'] = get_link_type(info)
# import ipdb; ipdb.set_trace()
yield info
def parse_bookmarks_export(html_file):
"""Parse netscape-format bookmarks export files (produced by all browsers)"""
@ -103,118 +126,17 @@ def parse_bookmarks_export(html_file):
match = pattern.search(line)
if match:
url = match.group(1)
secs = match.group(2)
dt = datetime.fromtimestamp(int(secs))
time = datetime.fromtimestamp(float(match.group(2)))
info = {
'url': url,
'domain': url.replace('http://', '').replace('https://', '').split('/', 1)[0],
'base_url': url.replace('https://', '').replace('http://', '').split('?', 1)[0],
'time': dt,
'timestamp': secs,
'domain': domain(url),
'base_url': base_url(url),
'timestamp': str(time.timestamp()),
'tags': "",
'title': match.group(3),
'sources': [html_file.name],
}
info['type'] = get_link_type(info)
yield info
def next_uniq_timestamp(used_timestamps, timestamp):
"""resolve duplicate timestamps by appending a decimal 1234, 1234 -> 1234.1, 1234.2"""
if timestamp not in used_timestamps:
return timestamp
if '.' in timestamp:
timestamp, nonce = timestamp.split('.')
nonce = int(nonce)
else:
nonce = 1
new_timestamp = '{}.{}'.format(timestamp, nonce)
while new_timestamp in used_timestamps:
nonce += 1
new_timestamp = '{}.{}'.format(timestamp, nonce)
return new_timestamp
def uniquefied_links(links):
"""uniqueify link timestamps by de-duping using url, returns links sorted most recent -> oldest
needed because firefox will produce exports where many links share the same timestamp, this func
ensures that all non-duplicate links have monotonically increasing timestamps
"""
links = list(reversed(sorted(links, key=lambda l: (l['timestamp'], l['url']))))
seen_timestamps = {}
for link in links:
t = link['timestamp']
if t in seen_timestamps:
if link['url'] == seen_timestamps[t]['url']:
# don't create new unique timestamp if link is the same
continue
else:
# resolve duplicate timstamp by appending a decimal
link['timestamp'] = next_uniq_timestamp(seen_timestamps, link['timestamp'])
seen_timestamps[link['timestamp']] = link
return links
def valid_links(links):
"""remove chrome://, about:// or other schemed links that cant be archived"""
return (link for link in links if link['url'].startswith('http') or link['url'].startswith('ftp'))
def html_appended_url(link):
"""calculate the path to the wgetted .html file, since wget may
adjust some paths to be different than the base_url path.
See docs on wget --adjust-extension."""
split_url = link['url'].split('#', 1)
query = ('%3F' + link['url'].split('?', 1)[-1]) if '?' in link['url'] else ''
if re.search(".+\\.[Hh][Tt][Mm][Ll]?$", split_url[0], re.I | re.M):
# already ends in .html
return link['base_url']
else:
# .html needs to be appended
without_scheme = split_url[0].split('://', 1)[-1].split('?', 1)[0]
if without_scheme.endswith('/'):
if query:
return '#'.join([without_scheme + 'index.html' + query + '.html', *split_url[1:]])
return '#'.join([without_scheme + 'index.html', *split_url[1:]])
else:
if query:
return '#'.join([without_scheme + 'index.html' + query + '.html', *split_url[1:]])
return '#'.join([without_scheme + '.html', *split_url[1:]])
def derived_link_info(link):
"""extend link info with the archive urls and other derived data"""
link_info = {
**link,
'date': str(link['time'])[:-3],
'google_favicon_url': 'https://www.google.com/s2/favicons?domain={domain}'.format(**link),
'favicon_url': 'archive/{timestamp}/favicon.ico'.format(**link),
'files_url': 'archive/{timestamp}/'.format(**link),
'archive_url': 'archive/{}/{}'.format(link['timestamp'], html_appended_url(link)),
'pdf_link': 'archive/{timestamp}/output.pdf'.format(**link),
'screenshot_link': 'archive/{timestamp}/screenshot.png'.format(**link),
'archive_org_url': 'https://web.archive.org/web/{base_url}'.format(**link),
}
# PDF and images are handled slightly differently
# wget, screenshot, & pdf urls all point to the same file
if link['type'] in ('PDF', 'image'):
link_info.update({
'archive_url': 'archive/{timestamp}/{base_url}'.format(**link),
'pdf_link': 'archive/{timestamp}/{base_url}'.format(**link),
'screenshot_link': 'archive/{timestamp}/{base_url}'.format(**link),
'title': '{title} ({type})'.format(**link),
})
return link_info

4
templates/index_row.html
View file

@ -1,10 +1,10 @@
<tr>
<td title="Date Bookmarked: $time">$date</td>
<td title="Bookmarked timestamp: $timestamp">$date</td>
<td><a href="$archive_url" style="font-size:1.4em;text-decoration:none;color:black;" title="$title">
<img src="$favicon_url">
$title <small style="background-color: #eee;border-radius:4px; float:right">$tags</small>
</td>
<td style="text-align:center"><a href="$files_url" title="Files">📂</a></td>
<td style="text-align:center"><a href="$files_url/index.html" title="Files">📂</a></td>
<td style="text-align:center"><a href="$pdf_link" title="PDF">📄</a></td>
<td style="text-align:center"><a href="$screenshot_link" title="Screenshot">🖼</a></td>
<td style="text-align:center"><a href="$archive_org_url" title="Archive.org">🏛</a></td>

258
templates/link_index.html Normal file
View file

@ -0,0 +1,258 @@
<html>
<head>
<meta charset="utf-8">
<title>$title</title>
<style>
html, body {
width: 100%;
height: 100%;
}
body {
background-color: #ddd;
}
header {
width: 100%;
background-color: #aa1e55;
margin: 0px;
text-align: center;
color: white;
}
header h1 {
padding-top: 5px;
padding-bottom: 5px;
margin: 0px;
font-weight: 200;
font-family: "Gill Sans", Helvetica, sans-serif;
}
.collapse-icon {
float: right;
color: black;
width: 126px;
font-size: 0.7em;
margin-top: 20px;
margin-right: -30px;
margin-left: -150px;
}
.nav-icon img {
display: inline-block;
margin-right: -200px;
float: left;
color: black;
height: 53px;
margin-top: 7px;
margin-left: 10px;
}
.nav-icon img:hover {
opacity: 0.5;
}
.title-url {
color: black;
}
.archive-page-header {
margin-top: 5px;
margin-bottom: 5px;
}
h1 small {
opacity: 0.4;
font-size: 0.6em;
}
h1 small:hover {
opacity: 0.8;
}
.card {
box-shadow: 2px 3px 14px 0px rgba(0,0,0,0.02);
}
.card h4 {
font-size: 1.4vw;
}
.card-body {
font-size: 1vw;
padding-top: 2vw;
padding-left: 1vw;
padding-right: 1vw;
padding-bottom: 5vw;
word-wrap: break-word;
max-height: 102px;
overflow: hidden;
}
.card-img-top {
border: 0px;
padding: 0px;
margin: 0px;
overflow: hidden;
opacity: 0.8;
border-top: 1px solid gray;
border-radius: 3px;
border-bottom: 1px solid #ddd;
height: 425px;
width: 400%;
margin-bottom: -330px;
transform: scale(0.25);
transform-origin: 0 0;
}
.full-page-iframe {
border-top: 1px solid #ddd;
width: 100%;
height: 69vh;
margin: 0px;
border: 0px;
border-top: 3px solid #aa1e55;
}
.card.selected-card {
border: 2px solid orange;
box-shadow: 0px -6px 13px 1px rgba(0,0,0,0.05);
}
.iframe-large {
height: 93%;
margin-top: -10px;
}
@media(max-width: 1092px) {
iframe {
display: none;
}
}
@media(max-width: 728px) {
.card h4 {
font-size: 5vw;
}
.card-body {
font-size: 4vw;
}
.card {
margin-bottom: 5px;
}
}
</style>
<script
src="https://code.jquery.com/jquery-3.2.1.slim.min.js"
integrity="sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g="
crossorigin="anonymous"></script>
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css" integrity="sha384-rwoIResjU2yc3z8GV/NPeZWAv56rSmLldC3R/AZzGRnGxQQKnKkoFVhFQhNUwEyJ" crossorigin="anonymous">
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/js/bootstrap.min.js" integrity="sha384-vBWWzlZJ8ea9aCX4pEW3rVHjgjt7zpkNpZk+02D9phzyeVkE+jo0ieGizqPLForn" crossorigin="anonymous"></script>
</head>
<body>
<header>
<h1 class="page-title">
<a href="#" class="collapse-icon" title="Collapse Navbar">
[-]
</a>
<a href="../../../index.html" class="nav-icon" title="Archived Sites">
<img src="https://nicksweeting.com/images/archive.png" alt="Archive Icon">
</a>
$title<br/>
<a href="$url" class="title-url">
<small><img src="$favicon" height="20px"> $base_url</small>
</a>
</h1>
</header>
<div class="site-header container-fluid">
<div class="row archive-page-header">
<div class="col-lg-2">
<div class="alert alert-warning">
Tags:
<span class="badge badge-success">$tags</span>
<span class="badge badge-default">$type</span>
<div style="height:5px"></div>
Bookmarked:<br/>
<small>$bookmarked<br/></small>
Archived:<br/>
<small>$updated</small>
<hr/>
<a href="index.json">JSON</a> | <a href=".">Files</a>
</div>
</div>
<div class="col-lg-2">
<div class="card selected-card">
<iframe class="card-img-top" src="$wget" sandbox="allow-same-origin allow-scripts allow-forms"></iframe>
<div class="card-body">
<a href="$wget" style="float:right"><small>➡️</small></a>
<a href="$wget" target="preview"><h4 class="card-title">Local Archive</h4></a>
<p class="card-text">archive/$domain</p>
</div>
</div>
</div>
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top" src="$pdf"></iframe>
<div class="card-body">
<a href="$pdf" style="float:right"><small>➡️</small></a>
<a href="$pdf" target="preview" id="pdf-btn"><h4 class="card-title">PDF</h4></a>
<p class="card-text">archive/output.pdf</p>
</div>
</div>
</div>
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top" src="$screenshot" sandbox="allow-same-origin allow-scripts allow-forms"></iframe>
<div class="card-body">
<a href="$screenshot" style="float:right"><small>➡️</small></a>
<a href="$screenshot" target="preview"><h4 class="card-title">Screenshot</h4></a>
<p class="card-text">archive/screenshot.png</p>
</div>
</div>
</div>
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top" src="$url" sandbox="allow-same-origin allow-scripts allow-forms"></iframe>
<div class="card-body">
<a href="$url" style="float:right"><small>➡️</small></a>
<a href="$url" target="preview"><h4 class="card-title">Original</h4></a>
<p class="card-text">$domain</p>
</div>
</div>
</div>
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top" src="$archive_org" sandbox="allow-same-origin allow-scripts allow-forms"></iframe>
<div class="card-body">
<a href="$archive_org" style="float:right"><small>➡️</small></a>
<a href="$archive_org" target="preview"><h4 class="card-title">Archive.Org</h4></a>
<p class="card-text">web.archive.org/web/...</p>
</div>
</div>
</div>
</div>
</div>
<iframe sandbox="allow-same-origin allow-scripts allow-forms" class="full-page-iframe" src="$wget" name="preview"></iframe>
</body>
<script>
jQuery('.card').on('click', function(e) {
jQuery('.selected-card').removeClass('selected-card')
jQuery(e.target).closest('.card').addClass('selected-card')
})
jQuery('.card a[target=preview]').on('click', function(e) {
if (e.currentTarget.href.endsWith('.pdf')) {
jQuery('.full-page-iframe')[0].removeAttribute('sandbox')
} else {
jQuery('.full-page-iframe')[0].sandbox = "allow-same-origin allow-scripts allow-forms"
}
return true
})
jQuery('iframe').map(function() {
if (this.src.endsWith('.pdf')) {
this.removeAttribute('sandbox')
this.src = this.src
}
})
jQuery('.collapse-icon').on('click', function() {
if (jQuery('.collapse-icon').text().includes('[-]')) {
jQuery('.collapse-icon').text('[+]')
jQuery('.site-header').hide()
jQuery('.full-page-iframe').addClass('iframe-large')
} else {
jQuery('.collapse-icon').text('[-]')
jQuery('.site-header').show()
jQuery('.full-page-iframe').removeClass('iframe-large')
}
return true
})
if (window.innerWidth < 1091) {
jQuery('.card a[target=preview]').attr('target', '_self')
}
</script>
</html>

216
util.py Normal file
View file

@ -0,0 +1,216 @@
import os
import sys
import time
import requests
from datetime import datetime
from subprocess import run, PIPE, DEVNULL
from multiprocessing import Process
from config import (
ARCHIVE_PERMISSIONS,
ARCHIVE_DIR,
TIMEOUT,
TERM_WIDTH,
SHOW_PROGRESS,
ANSI,
CHROME_BINARY,
FETCH_WGET,
FETCH_PDF,
FETCH_SCREENSHOT,
FETCH_FAVICON,
FETCH_AUDIO,
FETCH_VIDEO,
SUBMIT_ARCHIVE_DOT_ORG,
)
def check_dependencies():
"""Check that all necessary dependencies are installed, and have valid versions"""
print('[*] Checking Dependencies:')
python_vers = float('{}.{}'.format(sys.version_info.major, sys.version_info.minor))
if python_vers < 3.5:
print('{}[X] Python version is not new enough: {} (>3.5 is required){}'.format(ANSI['red'], python_vers, ANSI['reset']))
print(' See https://github.com/pirate/bookmark-archiver#troubleshooting for help upgrading your Python installation.')
raise SystemExit(1)
if FETCH_PDF or FETCH_SCREENSHOT:
if run(['which', CHROME_BINARY]).returncode:
print('{}[X] Missing dependency: {}{}'.format(ANSI['red'], CHROME_BINARY, ANSI['reset']))
print(' Run ./setup.sh, then confirm it was installed with: {} --version'.format(CHROME_BINARY))
print(' See https://github.com/pirate/bookmark-archiver for help.')
raise SystemExit(1)
# parse chrome --version e.g. Google Chrome 61.0.3114.0 canary / Chromium 59.0.3029.110 built on Ubuntu, running on Ubuntu 16.04
try:
result = run([CHROME_BINARY, '--version'], stdout=PIPE)
version = result.stdout.decode('utf-8').replace('Google Chrome ', '').replace('Chromium ', '').split(' ', 1)[0].split('.', 1)[0] # TODO: regex might be better
if int(version) < 59:
print('{red}[X] Chrome version must be 59 or greater for headless PDF and screenshot saving{reset}'.format(**ANSI))
print(' See https://github.com/pirate/bookmark-archiver for help.')
raise SystemExit(1)
except (TypeError, OSError):
print('{red}[X] Failed to parse Chrome version, is it installed properly?{reset}'.format(**ANSI))
print(' Run ./setup.sh, then confirm it was installed with: {} --version'.format(CHROME_BINARY))
print(' See https://github.com/pirate/bookmark-archiver for help.')
raise SystemExit(1)
if FETCH_WGET:
if run(['which', 'wget']).returncode or run(['wget', '--version'], stdout=DEVNULL).returncode:
print('{red}[X] Missing dependency: wget{reset}'.format(**ANSI))
print(' Run ./setup.sh, then confirm it was installed with: {} --version'.format('wget'))
print(' See https://github.com/pirate/bookmark-archiver for help.')
raise SystemExit(1)
if FETCH_FAVICON or SUBMIT_ARCHIVE_DOT_ORG:
if run(['which', 'curl']).returncode or run(['curl', '--version'], stdout=DEVNULL).returncode:
print('{red}[X] Missing dependency: curl{reset}'.format(**ANSI))
print(' Run ./setup.sh, then confirm it was installed with: {} --version'.format('curl'))
print(' See https://github.com/pirate/bookmark-archiver for help.')
raise SystemExit(1)
if FETCH_AUDIO or FETCH_VIDEO:
if run(['which', 'youtube-dl']).returncode or run(['youtube-dl', '--version'], stdout=DEVNULL).returncode:
print('{red}[X] Missing dependency: youtube-dl{reset}'.format(**ANSI))
print(' Run ./setup.sh, then confirm it was installed with: {} --version'.format('youtube-dl'))
print(' See https://github.com/pirate/bookmark-archiver for help.')
raise SystemExit(1)
def chmod_file(path, cwd='.', permissions=ARCHIVE_PERMISSIONS, timeout=30):
"""chmod -R <permissions> <cwd>/<path>"""
if not os.path.exists(os.path.join(cwd, path)):
raise Exception('Failed to chmod: {} does not exist (did the previous step fail?)'.format(path))
chmod_result = run(['chmod', '-R', permissions, path], cwd=cwd, stdout=DEVNULL, stderr=PIPE, timeout=timeout)
if chmod_result.returncode == 1:
print(' ', chmod_result.stderr.decode())
raise Exception('Failed to chmod {}/{}'.format(cwd, path))
def progress(seconds=TIMEOUT, prefix=''):
"""Show a (subprocess-controlled) progress bar with a <seconds> timeout,
returns end() function to instantly finish the progress
"""
if not SHOW_PROGRESS:
return lambda: None
chunk = '█' if sys.stdout.encoding == 'UTF-8' else '#'
chunks = TERM_WIDTH - len(prefix) - 20 # number of progress chunks to show (aka max bar width)
def progress_bar(seconds=seconds, prefix=prefix):
"""show timer in the form of progress bar, with percentage and seconds remaining"""
try:
for s in range(seconds * chunks):
progress = s / chunks / seconds * 100
bar_width = round(progress/(100/chunks))
# ████████████████████ 0.9% (1/60sec)
sys.stdout.write('\r{0}{1}{2}{3} {4}% ({5}/{6}sec)'.format(
prefix,
ANSI['green'],
(chunk * bar_width).ljust(chunks),
ANSI['reset'],
round(progress, 1),
round(s/chunks),
seconds,
))
sys.stdout.flush()
time.sleep(1 / chunks)
# ██████████████████████████████████ 100.0% (60/60sec)
sys.stdout.write('\r{0}{1}{2}{3} {4}% ({5}/{6}sec)\n'.format(
prefix,
ANSI['red'],
chunk * chunks,
ANSI['reset'],
100.0,
seconds,
seconds,
))
sys.stdout.flush()
except KeyboardInterrupt:
print()
pass
p = Process(target=progress_bar)
p.start()
def end():
"""immediately finish progress and clear the progressbar line"""
p.terminate()
sys.stdout.write('\r{}{}\r'.format((' ' * TERM_WIDTH), ANSI['reset'])) # clear whole terminal line
sys.stdout.flush()
return end
def download_url(url):
if not os.path.exists(os.path.join(ARCHIVE_DIR, 'downloads')):
os.makedirs(os.path.join(ARCHIVE_DIR, 'downloads'))
url_domain = url.split('/', 3)[2]
output_path = os.path.join(ARCHIVE_DIR, 'downloads', '{}.txt'.format(url_domain))
print('[*] [{}] Downloading {} > {}'.format(
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
url,
output_path,
))
end = progress(TIMEOUT, prefix=' ')
try:
downloaded_xml = requests.get(url).content.decode()
end()
except Exception as e:
end()
print('[!] Failed to download {}\n'.format(url))
print(' ', e)
raise SystemExit(1)
with open(output_path, 'w', encoding='utf-8') as f:
f.write(downloaded_xml)
return output_path
def get_str_between(string, start, end=None):
"""(<abc>12345</def>, <abc>, </def>) -> 12345"""
content = string.split(start, 1)[-1]
if end is not None:
content = content.rsplit(end, 1)[0]
return content
def get_link_type(link):
"""Certain types of links need to be handled specially, this figures out when that's the case"""
if link['base_url'].endswith('.pdf'):
return 'PDF'
elif link['base_url'].rsplit('.', 1) in ('pdf', 'png', 'jpg', 'jpeg', 'svg', 'bmp', 'gif', 'tiff', 'webp'):
return 'image'
elif 'wikipedia.org' in link['domain']:
return 'wiki'
elif 'youtube.com' in link['domain']:
return 'youtube'
elif 'soundcloud.com' in link['domain']:
return 'soundcloud'
elif 'youku.com' in link['domain']:
return 'youku'
elif 'vimeo.com' in link['domain']:
return 'vimeo'
return None
# URL helpers
without_scheme = lambda url: url.replace('http://', '').replace('https://', '').replace('ftp://', '')
without_query = lambda url: url.split('?', 1)[0]
without_hash = lambda url: url.split('#', 1)[0]
without_path = lambda url: url.split('/', 1)[0]
domain = lambda url: without_hash(without_query(without_path(without_scheme(url))))
base_url = lambda url: without_query(without_scheme(url))