diff --git a/archivebox/archive.py b/archivebox/archive.py index 8779f5cf..18d31023 100755 --- a/archivebox/archive.py +++ b/archivebox/archive.py @@ -180,7 +180,7 @@ def update_archive_data(import_path: Optional[str]=None, resume: Optional[float] all_links, new_links = load_links_index(out_dir=OUTPUT_DIR, import_path=import_path) # Step 2: Write updated index with deduped old and new links back to disk - write_links_index(out_dir=OUTPUT_DIR, links=list(all_links)) + write_links_index(links=list(all_links), out_dir=OUTPUT_DIR) # Step 3: Run the archive methods for each link links = new_links if ONLY_NEW else all_links @@ -189,7 +189,7 @@ def update_archive_data(import_path: Optional[str]=None, resume: Optional[float] link: Optional[Link] = None try: for idx, link in enumerate(links_after_timestamp(links, resume)): - archive_link(link) + archive_link(link, link_dir=link.link_dir) except KeyboardInterrupt: log_archiving_paused(len(links), idx, link.timestamp if link else '0') @@ -203,7 +203,7 @@ def update_archive_data(import_path: Optional[str]=None, resume: Optional[float] # Step 4: Re-write links index with updated titles, icons, and resources all_links, _ = load_links_index(out_dir=OUTPUT_DIR) - write_links_index(out_dir=OUTPUT_DIR, links=list(all_links), finished=True) + write_links_index(links=list(all_links), out_dir=OUTPUT_DIR, finished=True) return all_links if __name__ == '__main__': diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index 16ee3392..7a76df13 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -1,6 +1,6 @@ import os -from typing import Dict, List, Tuple +from typing import Dict, List, Tuple, Optional from collections import defaultdict from datetime import datetime @@ -69,7 +69,7 @@ class ArchiveError(Exception): @enforce_types -def archive_link(link: Link, page=None) -> Link: +def archive_link(link: Link, link_dir: Optional[str]=None) -> Link: """download the DOM, PDF, and a screenshot into a folder named after the link's timestamp""" ARCHIVE_METHODS = ( @@ -84,13 +84,14 @@ def archive_link(link: Link, page=None) -> Link: ('archive_org', should_fetch_archive_dot_org, archive_dot_org), ) + link_dir = link_dir or link.link_dir try: - is_new = not os.path.exists(link.link_dir) + is_new = not os.path.exists(link_dir) if is_new: - os.makedirs(link.link_dir) + os.makedirs(link_dir) - link = load_json_link_index(link.link_dir, link) - log_link_archiving_started(link.link_dir, link, is_new) + link = load_json_link_index(link, link_dir) + log_link_archiving_started(link, link_dir, is_new) link = link.overwrite(updated=datetime.now()) stats = {'skipped': 0, 'succeeded': 0, 'failed': 0} @@ -99,10 +100,10 @@ def archive_link(link: Link, page=None) -> Link: if method_name not in link.history: link.history[method_name] = [] - if should_run(link.link_dir, link): + if should_run(link, link_dir): log_archive_method_started(method_name) - result = method_function(link.link_dir, link) + result = method_function(link, link_dir) link.history[method_name].append(result) @@ -126,7 +127,7 @@ def archive_link(link: Link, page=None) -> Link: patch_links_index(link) - log_link_archiving_finished(link.link_dir, link, is_new, stats) + log_link_archiving_finished(link, link.link_dir, is_new, stats) except KeyboardInterrupt: raise @@ -141,7 +142,7 @@ def archive_link(link: Link, page=None) -> Link: ### Archive Method Functions @enforce_types -def should_fetch_title(link_dir: str, link: Link) -> bool: +def should_fetch_title(link: Link, link_dir: Optional[str]=None) -> bool: # if link already has valid title, skip it if link.title and not link.title.lower().startswith('http'): return False @@ -152,7 +153,7 @@ def should_fetch_title(link_dir: str, link: Link) -> bool: return FETCH_TITLE @enforce_types -def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_title(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """try to guess the page's title from its content""" output = None @@ -186,14 +187,14 @@ def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResul @enforce_types -def should_fetch_favicon(link_dir: str, link: Link) -> bool: +def should_fetch_favicon(link: Link, link_dir: Optional[str]=None) -> bool: if os.path.exists(os.path.join(link_dir, 'favicon.ico')): return False return FETCH_FAVICON @enforce_types -def fetch_favicon(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_favicon(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """download site favicon from google's favicon api""" output = 'favicon.ico' @@ -226,7 +227,7 @@ def fetch_favicon(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveRes ) @enforce_types -def should_fetch_wget(link_dir: str, link: Link) -> bool: +def should_fetch_wget(link: Link, link_dir: Optional[str]=None) -> bool: output_path = wget_output_path(link) if output_path and os.path.exists(os.path.join(link_dir, output_path)): return False @@ -235,7 +236,7 @@ def should_fetch_wget(link_dir: str, link: Link) -> bool: @enforce_types -def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_wget(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """download full site using wget""" if FETCH_WARC: @@ -315,7 +316,7 @@ def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult ) @enforce_types -def should_fetch_pdf(link_dir: str, link: Link) -> bool: +def should_fetch_pdf(link: Link, link_dir: Optional[str]=None) -> bool: if is_static_file(link.url): return False @@ -326,7 +327,7 @@ def should_fetch_pdf(link_dir: str, link: Link) -> bool: @enforce_types -def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_pdf(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """print PDF of site to file using chrome --headless""" output = 'output.pdf' @@ -361,7 +362,7 @@ def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: ) @enforce_types -def should_fetch_screenshot(link_dir: str, link: Link) -> bool: +def should_fetch_screenshot(link: Link, link_dir: Optional[str]=None) -> bool: if is_static_file(link.url): return False @@ -371,7 +372,7 @@ def should_fetch_screenshot(link_dir: str, link: Link) -> bool: return FETCH_SCREENSHOT @enforce_types -def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_screenshot(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """take screenshot of site using chrome --headless""" output = 'screenshot.png' @@ -406,7 +407,7 @@ def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> Archive ) @enforce_types -def should_fetch_dom(link_dir: str, link: Link) -> bool: +def should_fetch_dom(link: Link, link_dir: Optional[str]=None) -> bool: if is_static_file(link.url): return False @@ -416,7 +417,7 @@ def should_fetch_dom(link_dir: str, link: Link) -> bool: return FETCH_DOM @enforce_types -def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_dom(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """print HTML of site to file using chrome --dump-html""" output = 'output.html' @@ -453,7 +454,7 @@ def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: ) @enforce_types -def should_fetch_git(link_dir: str, link: Link) -> bool: +def should_fetch_git(link: Link, link_dir: Optional[str]=None) -> bool: if is_static_file(link.url): return False @@ -471,7 +472,7 @@ def should_fetch_git(link_dir: str, link: Link) -> bool: @enforce_types -def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_git(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """download full site using git""" output = 'git' @@ -514,7 +515,7 @@ def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: @enforce_types -def should_fetch_media(link_dir: str, link: Link) -> bool: +def should_fetch_media(link: Link, link_dir: Optional[str]=None) -> bool: if is_static_file(link.url): return False @@ -524,7 +525,7 @@ def should_fetch_media(link_dir: str, link: Link) -> bool: return FETCH_MEDIA @enforce_types -def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult: +def fetch_media(link: Link, link_dir: Optional[str]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult: """Download playlists or individual video, audio, and subtitles using youtube-dl""" output = 'media' @@ -588,7 +589,7 @@ def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> Archiv @enforce_types -def should_fetch_archive_dot_org(link_dir: str, link: Link) -> bool: +def should_fetch_archive_dot_org(link: Link, link_dir: Optional[str]=None) -> bool: if is_static_file(link.url): return False @@ -599,7 +600,7 @@ def should_fetch_archive_dot_org(link_dir: str, link: Link) -> bool: return SUBMIT_ARCHIVE_DOT_ORG @enforce_types -def archive_dot_org(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def archive_dot_org(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """submit site to archive.org for archiving via their service, save returned archive url""" output = 'archive.org.txt' diff --git a/archivebox/index.py b/archivebox/index.py index 66b234a2..d3fe7965 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -39,23 +39,25 @@ from .logs import ( TITLE_LOADING_MSG = 'Not yet archived...' + + ### Homepage index for all the links @enforce_types -def write_links_index(out_dir: str, links: List[Link], finished: bool=False) -> None: +def write_links_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=False) -> None: """create index.html file for a given list of links""" log_indexing_process_started() log_indexing_started(out_dir, 'index.json') timer = TimedProgress(TIMEOUT * 2, prefix=' ') - write_json_links_index(out_dir, links) + write_json_links_index(links, out_dir=out_dir) timer.end() log_indexing_finished(out_dir, 'index.json') log_indexing_started(out_dir, 'index.html') timer = TimedProgress(TIMEOUT * 2, prefix=' ') - write_html_links_index(out_dir, links, finished=finished) + write_html_links_index(links, out_dir=out_dir, finished=finished) timer.end() log_indexing_finished(out_dir, 'index.html') @@ -87,7 +89,7 @@ def load_links_index(out_dir: str=OUTPUT_DIR, import_path: Optional[str]=None) - @enforce_types -def write_json_links_index(out_dir: str, links: List[Link]) -> None: +def write_json_links_index(links: List[Link], out_dir: str=OUTPUT_DIR) -> None: """write the json link index to a given path""" assert isinstance(links, List), 'Links must be a list, not a generator.' @@ -199,7 +201,6 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None: successful = link.num_outputs # Patch JSON index - changed = False json_file_links = parse_json_links_index(out_dir) patched_links = [] for saved_link in json_file_links: @@ -212,7 +213,7 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None: else: patched_links.append(saved_link) - write_json_links_index(out_dir, patched_links) + write_json_links_index(patched_links, out_dir=out_dir) # Patch HTML index html_path = os.path.join(out_dir, 'index.html') @@ -231,27 +232,27 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None: ### Individual link index @enforce_types -def write_link_index(out_dir: str, link: Link) -> None: - write_json_link_index(out_dir, link) - write_html_link_index(out_dir, link) +def write_link_index(link: Link, link_dir: Optional[str]=None) -> None: + link_dir = link_dir or link.link_dir + + write_json_link_index(link, link_dir) + write_html_link_index(link, link_dir) @enforce_types -def write_json_link_index(out_dir: str, link: Link) -> None: +def write_json_link_index(link: Link, link_dir: Optional[str]=None) -> None: """write a json file with some info about the link""" - path = os.path.join(out_dir, 'index.json') - - with open(path, 'w', encoding='utf-8') as f: - json.dump(link._asdict(), f, indent=4, cls=ExtendedEncoder) + link_dir = link_dir or link.link_dir + path = os.path.join(link_dir, 'index.json') chmod_file(path) @enforce_types -def parse_json_link_index(out_dir: str) -> Optional[Link]: +def parse_json_link_index(link_dir: str) -> Optional[Link]: """load the json link index from a given directory""" - existing_index = os.path.join(out_dir, 'index.json') + existing_index = os.path.join(link_dir, 'index.json') if os.path.exists(existing_index): with open(existing_index, 'r', encoding='utf-8') as f: link_json = json.load(f) @@ -260,18 +261,21 @@ def parse_json_link_index(out_dir: str) -> Optional[Link]: @enforce_types -def load_json_link_index(out_dir: str, link: Link) -> Link: +def load_json_link_index(link: Link, link_dir: Optional[str]=None) -> Link: """check for an existing link archive in the given directory, and load+merge it into the given link dict """ - existing_link = parse_json_link_index(out_dir) + link_dir = link_dir or link.link_dir + existing_link = parse_json_link_index(link_dir) if existing_link: return merge_links(existing_link, link) return link @enforce_types -def write_html_link_index(out_dir: str, link: Link) -> None: +def write_html_link_index(link: Link, link_dir: Optional[str]=None) -> None: + link_dir = link_dir or link.link_dir + with open(os.path.join(TEMPLATES_DIR, 'link_index.html'), 'r', encoding='utf-8') as f: link_html = f.read() diff --git a/archivebox/logs.py b/archivebox/logs.py index 4e21731b..155f81e6 100644 --- a/archivebox/logs.py +++ b/archivebox/logs.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Optional from .schema import Link, ArchiveResult -from .config import ANSI, REPO_DIR, OUTPUT_DIR +from .config import ANSI, OUTPUT_DIR @dataclass @@ -17,14 +17,14 @@ class RuntimeStats: succeeded: int = 0 failed: int = 0 - parse_start_ts: datetime = None - parse_end_ts: datetime = None + parse_start_ts: Optional[datetime] = None + parse_end_ts: Optional[datetime] = None - index_start_ts: datetime = None - index_end_ts: datetime = None + index_start_ts: Optional[datetime] = None + index_end_ts: Optional[datetime] = None - archiving_start_ts: datetime = None - archiving_end_ts: datetime = None + archiving_start_ts: Optional[datetime] = None + archiving_end_ts: Optional[datetime] = None # globals are bad, mmkay _LAST_RUN_STATS = RuntimeStats() @@ -131,7 +131,7 @@ def log_archiving_finished(num_links: int): print(' {}/index.html'.format(OUTPUT_DIR)) -def log_link_archiving_started(link_dir: str, link: Link, is_new: bool): +def log_link_archiving_started(link: Link, link_dir: str, is_new: bool): # [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford" # http://www.benstopford.com/2015/02/14/log-structured-merge-trees/ # > output/archive/1478739709 @@ -149,7 +149,7 @@ def log_link_archiving_started(link_dir: str, link: Link, is_new: bool): pretty_path(link_dir), )) -def log_link_archiving_finished(link_dir: str, link: Link, is_new: bool, stats: dict): +def log_link_archiving_finished(link: Link, link_dir: str, is_new: bool, stats: dict): total = sum(stats.values()) if stats['failed'] > 0 : diff --git a/archivebox/util.py b/archivebox/util.py index fe3c57cf..dd71eecd 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -1,11 +1,12 @@ import os import re import sys +import json import time import shutil from json import JSONEncoder -from typing import List, Optional, Any +from typing import List, Optional, Any, Union from inspect import signature, _empty from functools import wraps from hashlib import sha256