mirror of
https://github.com/ArchiveBox/ArchiveBox
synced 2024-11-10 06:34:16 +00:00
initial commiit
This commit is contained in:
Nick Sweeting
parent
61f6f02b59
commit
206b5fc57f
5 changed files with 295 additions and 0 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -1,3 +1,6 @@
|
|||
# Pocket archive output folder
|
||||
pocket/
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
|
|
|
|||
165
archive.py
Executable file
165
archive.py
Executable file
|
|
@ -0,0 +1,165 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
|
||||
# sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list'
|
||||
# apt update; apt install google-chrome-beta
|
||||
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
|
||||
from datetime import datetime
|
||||
from subprocess import run, DEVNULL
|
||||
|
||||
|
||||
RESOLUTION = '1440,900'
|
||||
|
||||
|
||||
def parse_pocket_export(html):
|
||||
pattern = re.compile("^\\s*<li><a href=\"(.+)\" time_added=\"(\\d+)\" tags=\"(.*)\">(.+)</a></li>", re.UNICODE)
|
||||
for line in html:
|
||||
match = pattern.search(line)
|
||||
if match:
|
||||
yield {
|
||||
'url': match.group(1).replace('http://www.readability.com/read?url=', ''),
|
||||
'domain': match.group(1).replace('http://', '').replace('https://', '').split('/')[0],
|
||||
'base_url': match.group(1).replace('https://', '').replace('http://', '').split('?')[0],
|
||||
'time': datetime.fromtimestamp(int(match.group(2))),
|
||||
'timestamp': match.group(2),
|
||||
'tags': match.group(3),
|
||||
'title': match.group(4).replace(' — Readability', '').replace('http://www.readability.com/read?url=', ''),
|
||||
}
|
||||
|
||||
def dump_index(links):
|
||||
with open('index_template.html', 'r') as f:
|
||||
index_html = f.read()
|
||||
|
||||
link_html = """\
|
||||
<tr>
|
||||
<td>{time}</td>
|
||||
<td><a href="archive/{timestamp}/{base_url}" style="font-size:1.4em;text-decoration:none;color:black;" title="{title}">
|
||||
<img src="archive/{timestamp}/favicon.ico">
|
||||
{title}
|
||||
</td>
|
||||
<td style="text-align:center"><a href="archive/{timestamp}/" title="Files">📂</a></td>
|
||||
<td style="text-align:center"><a href="archive/{timestamp}/output.pdf" title="PDF">📄</a></td>
|
||||
<td style="text-align:center"><a href="archive/{timestamp}/screenshot.png" title="Screenshot">🖼</a></td>
|
||||
<td>🔗 <img src="https://www.google.com/s2/favicons?domain={domain}" height="16px"> <a href="{url}">{url}</a></td>
|
||||
</tr>"""
|
||||
|
||||
with open('pocket/index.html', 'w') as f:
|
||||
article_rows = '\n'.join(
|
||||
link_html.format(**link) for link in links
|
||||
)
|
||||
f.write(index_html.format(datetime.now().strftime('%Y-%m-%d %H:%M'), article_rows))
|
||||
|
||||
|
||||
def dump_website(link, overwrite=False):
|
||||
"""download the DOM, PDF, and a screenshot into a folder named after the link's timestamp"""
|
||||
|
||||
print('[+] [{time}] Archiving "{title}": {url}'.format(**link))
|
||||
|
||||
out_dir = 'pocket/archive/{timestamp}'.format(**link)
|
||||
if not os.path.exists(out_dir):
|
||||
os.makedirs(out_dir)
|
||||
|
||||
if link['base_url'].endswith('.pdf'):
|
||||
print(' i PDF File')
|
||||
elif 'youtube.com' in link['domain']:
|
||||
print(' i Youtube Video')
|
||||
elif 'wikipedia.org' in link['domain']:
|
||||
print(' i Wikipedia Article')
|
||||
|
||||
# download full site
|
||||
if not os.path.exists('{}/{}'.format(out_dir, link['domain'])) or overwrite:
|
||||
print(' - Downloading Full Site')
|
||||
CMD = [
|
||||
*'wget --no-clobber --page-requisites --adjust-extension --convert-links --no-parent'.split(' '),
|
||||
link['url'],
|
||||
]
|
||||
try:
|
||||
proc = run(CMD, stdout=DEVNULL, stderr=DEVNULL, cwd=out_dir, timeout=20) # dom.html
|
||||
except Exception as e:
|
||||
print(' Exception: {}'.format(e.__class__.__name__))
|
||||
else:
|
||||
print(' √ Skipping site download')
|
||||
|
||||
# download PDF
|
||||
if (not os.path.exists('{}/output.pdf'.format(out_dir)) or overwrite) and not link['base_url'].endswith('.pdf'):
|
||||
print(' - Printing PDF')
|
||||
CMD = 'google-chrome --headless --disable-gpu --print-to-pdf'.split(' ')
|
||||
try:
|
||||
proc = run([*CMD, link['url']], stdout=DEVNULL, stderr=DEVNULL, cwd=out_dir, timeout=20) # output.pdf
|
||||
except Exception as e:
|
||||
print(' Exception: {}'.format(e.__class__.__name__))
|
||||
else:
|
||||
print(' √ Skipping PDF print')
|
||||
|
||||
# take screenshot
|
||||
if (not os.path.exists('{}/screenshot.png'.format(out_dir)) or overwrite) and not link['base_url'].endswith('.pdf'):
|
||||
print(' - Snapping Screenshot')
|
||||
CMD = 'google-chrome --headless --disable-gpu --screenshot'.split(' ')
|
||||
try:
|
||||
proc = run([*CMD, '--window-size={}'.format(RESOLUTION), link['url']], stdout=DEVNULL, stderr=DEVNULL, cwd=out_dir, timeout=20) # sreenshot.png
|
||||
except Exception as e:
|
||||
print(' Exception: {}'.format(e.__class__.__name__))
|
||||
else:
|
||||
print(' √ Skipping screenshot')
|
||||
|
||||
# download favicon
|
||||
if not os.path.exists('{}/favicon.ico'.format(out_dir)) or overwrite:
|
||||
print(' - Fetching Favicon')
|
||||
CMD = 'curl https://www.google.com/s2/favicons?domain={domain}'.format(**link).split(' ')
|
||||
fout = open('{}/favicon.ico'.format(out_dir), 'w')
|
||||
try:
|
||||
proc = run([*CMD], stdout=fout, stderr=DEVNULL, cwd=out_dir, timeout=20) # dom.html
|
||||
except Exception as e:
|
||||
print(' Exception: {}'.format(e.__class__.__name__))
|
||||
fout.close()
|
||||
else:
|
||||
print(' √ Skipping favicon')
|
||||
|
||||
run(['chmod', '-R', '755', out_dir], timeout=1)
|
||||
|
||||
def create_archive(pocket_file, resume=None):
|
||||
print('[+] [{}] Starting pocket archive from {}'.format(datetime.now(), pocket_file))
|
||||
|
||||
if not os.path.exists('pocket'):
|
||||
os.makedirs('pocket')
|
||||
|
||||
if not os.path.exists('pocket/archive'):
|
||||
os.makedirs('pocket/archive')
|
||||
|
||||
with open(pocket_file, 'r', encoding='utf-8') as f:
|
||||
links = parse_pocket_export(f)
|
||||
links = list(reversed(sorted(links, key=lambda l: l['timestamp']))) # most recent first
|
||||
if resume:
|
||||
links = [link for link in links if link['timestamp'] >= resume]
|
||||
|
||||
if not links:
|
||||
print('[X] No links found in {}'.format(pocket_file))
|
||||
raise SystemExit(1)
|
||||
|
||||
dump_index(links)
|
||||
|
||||
run(['chmod', '-R', '755', 'pocket'], timeout=1)
|
||||
|
||||
print('[*] [{}] Created archive index.'.format(datetime.now()))
|
||||
|
||||
for link in links:
|
||||
dump_website(link)
|
||||
|
||||
print('[√] [{}] Archive complete.'.format(datetime.now()))
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pocket_file = 'ril_export.html'
|
||||
resume = None
|
||||
try:
|
||||
pocket_file = sys.argv[1]
|
||||
resume = sys.argv[2]
|
||||
except IndexError:
|
||||
pass
|
||||
|
||||
create_archive(pocket_file, resume=resume)
|
||||
37
example_ril_export.html
Normal file
37
example_ril_export.html
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
<!--So long and thanks for all the fish-->
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
|
||||
<title>Pocket Export</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Unread</h1>
|
||||
<ul>
|
||||
<li><a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3110382/" time_added="1493913054" tags="">The Radical Plasticity Thesis: How the Brain Learns to be Conscious</a></li>
|
||||
<li><a href="https://martinfowler.com/eaaDev/uiArchs.html" time_added="1493909628" tags="">GUI Architectures</a></li>
|
||||
<li><a href="https://issuu.com/crowdcraft/docs/shanghai-talk-july-2012" time_added="1493900327" tags="make512">Shanghai Talk July 2012 by Mike Hall - issuu</a></li>
|
||||
<li><a href="http://make512.weebly.com/about-us.html" time_added="1493900002" tags="">About Us - make512</a></li>
|
||||
<li><a href="https://openzfsonosx.org/wiki/ZFS_on_Boot" time_added="1493887140" tags="">ZFS on Boot - OpenZFS on OS X</a></li>
|
||||
<li><a href="http://www.softpanorama.org/DNS/history.shtml" time_added="1493869958" tags="">History of DNS</a></li>
|
||||
<li><a href="https://chromium.googlesource.com/chromium/src/+/master/docs/linux_sandboxing.md" time_added="1493869649" tags="">Linux Sandboxing</a></li>
|
||||
<li><a href="https://hackernoon.com/rems-and-ems-and-why-you-probably-dont-need-them-664b9ce1e09f" time_added="1493694979" tags="">rems and ems, and why you probably don’t need them – Hacker Noon</a></li>
|
||||
<li><a href="https://wiki.archlinux.org/index.php/full_system_backup_with_rsync" time_added="1493581911" tags="">Full system backup with rsync - ArchWiki</a></li>
|
||||
</ul>
|
||||
|
||||
<h1>Read Archive</h1>
|
||||
<ul>
|
||||
<li><a href="https://github.com/Droogans/unmaintainable-code" time_added="1478739800" tags="">Droogans/unmaintainable-code: An easier to share version of the infamous ht</a></li>
|
||||
<li><a href="http://www.benstopford.com/2015/02/14/log-structured-merge-trees/" time_added="1478739709" tags="">Log Structured Merge Trees - ben stopford</a></li>
|
||||
<li><a href="http://jgthms.com/web-design-in-4-minutes/#share" time_added="1478739628" tags="">Web Design in 4 minutes</a></li>
|
||||
<li><a href="https://eev.ee/blog/2016/07/26/the-hardest-problem-in-computer-science/" time_added="1478739622" tags="">The hardest problem in computer science / fuzzy notepad</a></li>
|
||||
<li><a href="https://medium.com/@iamjordanlittle/9-underutilized-features-in-css-90ced6ddbfe7#.690ah7whf" time_added="1476686912" tags="">9 Underutilized Features in CSS – Medium</a></li>
|
||||
<li><a href="http://themacro.com/articles/2016/09/employee-1-coinbase/" time_added="1476686907" tags="">Employee #1: Coinbase · The Macro</a></li>
|
||||
<li><a href="https://juokaz.com/blog/becoming-a-cto" time_added="1476686904" tags="">Becoming a CTO // Juozas Kaziukėnas</a></li>
|
||||
<li><a href="https://backchannel.com/the-internet-really-has-changed-everything-here-s-the-proof-928eaead18a8#.ekfmwcjh2" time_added="1476686896" tags="">The Internet Really Has Changed Everything. Here’s the Proof.</a></li>
|
||||
<li><a href="http://www.hindawi.com/journals/ijbm/2011/172389/" time_added="1424321329" tags="">Experimental and Modeling Study of Collagen Scaffolds with the Effects of C</a></li>
|
||||
<li><a href="http://search.cpan.org/dist/Locale-Maketext/lib/Locale/Maketext/TPJ13.pod?#A_Localization_Horror_Story:_It_Could_Happen_To_You" time_added="1424306906" tags="">Locale::Maketext::TPJ13 - search.cpan.org</a></li>
|
||||
|
||||
</ul>
|
||||
</body>
|
||||
</html>
|
||||
90
index_template.html
Normal file
90
index_template.html
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
<html lang="en">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
|
||||
<title>Archived Sites</title>
|
||||
<style>
|
||||
html, body {{
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
font-size: 20px;
|
||||
font-weight: 200;
|
||||
text-align: center;
|
||||
margin: 0px;
|
||||
padding: 0px;
|
||||
font-family: "Gill Sans", Helvetica, sans-serif;
|
||||
}}
|
||||
header {{
|
||||
background-color: #aa1e55;
|
||||
color: white;
|
||||
padding: 10px;
|
||||
}}
|
||||
header h1 {{
|
||||
font-weight: 300;
|
||||
color: black;
|
||||
margin-top: 10px;
|
||||
margin-bottom: 12px;
|
||||
}}
|
||||
header h1 small {{
|
||||
color: white;
|
||||
font-size:0.5em;
|
||||
}}
|
||||
header h1 small a {{
|
||||
text-decoration: none;
|
||||
color: orange;
|
||||
opacity: 0.6
|
||||
font-weight: 300;
|
||||
}}
|
||||
header h1 small a:hover {{
|
||||
opacity: 1;
|
||||
}}
|
||||
table {{
|
||||
padding: 6px;
|
||||
width: 100%;
|
||||
}}
|
||||
table thead th {{
|
||||
font-weight: 400;
|
||||
}}
|
||||
tbody tr:nth-child(odd) {{
|
||||
background-color: #ffebeb;
|
||||
}}
|
||||
table tr td {{
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
padding-bottom: 0.4em;
|
||||
padding-top: 0.4em;
|
||||
padding-left: 2px;
|
||||
}}
|
||||
table tr td img {{
|
||||
height: 24px;
|
||||
padding: 0px;
|
||||
padding-right: 5px;
|
||||
text-indent: -10000px;
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1 title="Last modified {}">
|
||||
<img src="https://nicksweeting.com/images/archive.png" height="36px">
|
||||
Archived Sites <img src="https://getpocket.com/favicon.ico" height="36px"> <br/>
|
||||
<small>
|
||||
Via: <a href="https://getpocket.com/export">getpocket.com/export</a> + <a href="https://github.com/pirate/pocket-archive-stream">archive_pocket.py</a>
|
||||
| <a href="https://getpocket.com/users/USERNAME/feed/all">RSS Feed</a>
|
||||
</small>
|
||||
</h1>
|
||||
</header>
|
||||
<table style="width:100%;height: 90%; overflow-y: scroll;table-layout: fixed">
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="width: 140px;"><img src="https://getpocket.com/favicon.ico" height="12px"> Pocketed Date</th>
|
||||
<th style="width: 45vw;">Saved Article</th>
|
||||
<th style="width: 50px">Files</th>
|
||||
<th style="width: 50px">PDF</th>
|
||||
<th style="width: 80px">Screenshot</th>
|
||||
<th style="width: 100px;whitespace:nowrap;overflow-x:scroll;display:block">Original URL</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>{}</tbody>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
BIN
screenshot.png
Normal file
BIN
screenshot.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 783 KiB |
Loading…
Reference in a new issue