make everything take link_dir as an optional arg since its derivable from link url

archivebox/archive.py

@@ -180,7 +180,7 @@ def update_archive_data(import_path: Optional[str]=None, resume: Optional[float]
     all_links, new_links = load_links_index(out_dir=OUTPUT_DIR, import_path=import_path)
 
     # Step 2: Write updated index with deduped old and new links back to disk
-    write_links_index(out_dir=OUTPUT_DIR, links=list(all_links))
+    write_links_index(links=list(all_links), out_dir=OUTPUT_DIR)
 
     # Step 3: Run the archive methods for each link
     links = new_links if ONLY_NEW else all_links
@@ -189,7 +189,7 @@ def update_archive_data(import_path: Optional[str]=None, resume: Optional[float]
     link: Optional[Link] = None
     try:
         for idx, link in enumerate(links_after_timestamp(links, resume)):
-            archive_link(link)
+            archive_link(link, link_dir=link.link_dir)
 
     except KeyboardInterrupt:
         log_archiving_paused(len(links), idx, link.timestamp if link else '0')
@@ -203,7 +203,7 @@ def update_archive_data(import_path: Optional[str]=None, resume: Optional[float]
 
     # Step 4: Re-write links index with updated titles, icons, and resources
     all_links, _ = load_links_index(out_dir=OUTPUT_DIR)
-    write_links_index(out_dir=OUTPUT_DIR, links=list(all_links), finished=True)
+    write_links_index(links=list(all_links), out_dir=OUTPUT_DIR, finished=True)
     return all_links
 
 if __name__ == '__main__':

archivebox/archive_methods.py

@@ -1,6 +1,6 @@
 import os
 
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Optional
 from collections import defaultdict
 from datetime import datetime
 
@@ -69,7 +69,7 @@ class ArchiveError(Exception):
 
 
 @enforce_types
-def archive_link(link: Link, page=None) -> Link:
+def archive_link(link: Link, link_dir: Optional[str]=None) -> Link:
     """download the DOM, PDF, and a screenshot into a folder named after the link's timestamp"""
 
     ARCHIVE_METHODS = (
@@ -84,13 +84,14 @@ def archive_link(link: Link, page=None) -> Link:
         ('archive_org', should_fetch_archive_dot_org, archive_dot_org),
     )
     
+    link_dir = link_dir or link.link_dir
     try:
-        is_new = not os.path.exists(link.link_dir)
+        is_new = not os.path.exists(link_dir)
         if is_new:
-            os.makedirs(link.link_dir)
+            os.makedirs(link_dir)
 
-        link = load_json_link_index(link.link_dir, link)
+        link = load_json_link_index(link, link_dir)
-        log_link_archiving_started(link.link_dir, link, is_new)
+        log_link_archiving_started(link, link_dir, is_new)
         link = link.overwrite(updated=datetime.now())
         stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
 
@@ -99,10 +100,10 @@ def archive_link(link: Link, page=None) -> Link:
                 if method_name not in link.history:
                     link.history[method_name] = []
                 
-                if should_run(link.link_dir, link):
+                if should_run(link, link_dir):
                     log_archive_method_started(method_name)
 
-                    result = method_function(link.link_dir, link)
+                    result = method_function(link, link_dir)
 
                     link.history[method_name].append(result)
 
@@ -126,7 +127,7 @@ def archive_link(link: Link, page=None) -> Link:
             patch_links_index(link)
 
 
-        log_link_archiving_finished(link.link_dir, link, is_new, stats)
+        log_link_archiving_finished(link, link.link_dir, is_new, stats)
 
     except KeyboardInterrupt:
         raise
@@ -141,7 +142,7 @@ def archive_link(link: Link, page=None) -> Link:
 ### Archive Method Functions
 
 @enforce_types
-def should_fetch_title(link_dir: str, link: Link) -> bool:
+def should_fetch_title(link: Link, link_dir: Optional[str]=None) -> bool:
     # if link already has valid title, skip it
     if link.title and not link.title.lower().startswith('http'):
         return False
@@ -152,7 +153,7 @@ def should_fetch_title(link_dir: str, link: Link) -> bool:
     return FETCH_TITLE
 
 @enforce_types
-def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
+def fetch_title(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """try to guess the page's title from its content"""
 
     output = None
@@ -186,14 +187,14 @@ def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResul
 
 
 @enforce_types
-def should_fetch_favicon(link_dir: str, link: Link) -> bool:
+def should_fetch_favicon(link: Link, link_dir: Optional[str]=None) -> bool:
     if os.path.exists(os.path.join(link_dir, 'favicon.ico')):
         return False
 
     return FETCH_FAVICON
     
 @enforce_types
-def fetch_favicon(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
+def fetch_favicon(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download site favicon from google's favicon api"""
 
     output = 'favicon.ico'
@@ -226,7 +227,7 @@ def fetch_favicon(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveRes
     )
 
 @enforce_types
-def should_fetch_wget(link_dir: str, link: Link) -> bool:
+def should_fetch_wget(link: Link, link_dir: Optional[str]=None) -> bool:
     output_path = wget_output_path(link)
     if output_path and os.path.exists(os.path.join(link_dir, output_path)):
         return False
@@ -235,7 +236,7 @@ def should_fetch_wget(link_dir: str, link: Link) -> bool:
 
 
 @enforce_types
-def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
+def fetch_wget(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download full site using wget"""
 
     if FETCH_WARC:
@@ -315,7 +316,7 @@ def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult
     )
 
 @enforce_types
-def should_fetch_pdf(link_dir: str, link: Link) -> bool:
+def should_fetch_pdf(link: Link, link_dir: Optional[str]=None) -> bool:
     if is_static_file(link.url):
         return False
     
@@ -326,7 +327,7 @@ def should_fetch_pdf(link_dir: str, link: Link) -> bool:
 
 
 @enforce_types
-def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
+def fetch_pdf(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """print PDF of site to file using chrome --headless"""
 
     output = 'output.pdf'
@@ -361,7 +362,7 @@ def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
     )
 
 @enforce_types
-def should_fetch_screenshot(link_dir: str, link: Link) -> bool:
+def should_fetch_screenshot(link: Link, link_dir: Optional[str]=None) -> bool:
     if is_static_file(link.url):
         return False
     
@@ -371,7 +372,7 @@ def should_fetch_screenshot(link_dir: str, link: Link) -> bool:
     return FETCH_SCREENSHOT
 
 @enforce_types
-def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
+def fetch_screenshot(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """take screenshot of site using chrome --headless"""
 
     output = 'screenshot.png'
@@ -406,7 +407,7 @@ def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> Archive
     )
 
 @enforce_types
-def should_fetch_dom(link_dir: str, link: Link) -> bool:
+def should_fetch_dom(link: Link, link_dir: Optional[str]=None) -> bool:
     if is_static_file(link.url):
         return False
     
@@ -416,7 +417,7 @@ def should_fetch_dom(link_dir: str, link: Link) -> bool:
     return FETCH_DOM
     
 @enforce_types
-def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
+def fetch_dom(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """print HTML of site to file using chrome --dump-html"""
 
     output = 'output.html'
@@ -453,7 +454,7 @@ def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
     )
 
 @enforce_types
-def should_fetch_git(link_dir: str, link: Link) -> bool:
+def should_fetch_git(link: Link, link_dir: Optional[str]=None) -> bool:
     if is_static_file(link.url):
         return False
 
@@ -471,7 +472,7 @@ def should_fetch_git(link_dir: str, link: Link) -> bool:
 
 
 @enforce_types
-def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
+def fetch_git(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download full site using git"""
 
     output = 'git'
@@ -514,7 +515,7 @@ def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
 
 
 @enforce_types
-def should_fetch_media(link_dir: str, link: Link) -> bool:
+def should_fetch_media(link: Link, link_dir: Optional[str]=None) -> bool:
     if is_static_file(link.url):
         return False
 
@@ -524,7 +525,7 @@ def should_fetch_media(link_dir: str, link: Link) -> bool:
     return FETCH_MEDIA
 
 @enforce_types
-def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult:
+def fetch_media(link: Link, link_dir: Optional[str]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult:
     """Download playlists or individual video, audio, and subtitles using youtube-dl"""
 
     output = 'media'
@@ -588,7 +589,7 @@ def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> Archiv
 
 
 @enforce_types
-def should_fetch_archive_dot_org(link_dir: str, link: Link) -> bool:
+def should_fetch_archive_dot_org(link: Link, link_dir: Optional[str]=None) -> bool:
     if is_static_file(link.url):
         return False
 
@@ -599,7 +600,7 @@ def should_fetch_archive_dot_org(link_dir: str, link: Link) -> bool:
     return SUBMIT_ARCHIVE_DOT_ORG
 
 @enforce_types
-def archive_dot_org(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
+def archive_dot_org(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """submit site to archive.org for archiving via their service, save returned archive url"""
 
     output = 'archive.org.txt'

archivebox/index.py

@@ -39,23 +39,25 @@ from .logs import (
 TITLE_LOADING_MSG = 'Not yet archived...'
 
 
+
+
 ### Homepage index for all the links
 
 @enforce_types
-def write_links_index(out_dir: str, links: List[Link], finished: bool=False) -> None:
+def write_links_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=False) -> None:
     """create index.html file for a given list of links"""
 
     log_indexing_process_started()
 
     log_indexing_started(out_dir, 'index.json')
     timer = TimedProgress(TIMEOUT * 2, prefix='      ')
-    write_json_links_index(out_dir, links)
+    write_json_links_index(links, out_dir=out_dir)
     timer.end()
     log_indexing_finished(out_dir, 'index.json')
     
     log_indexing_started(out_dir, 'index.html')
     timer = TimedProgress(TIMEOUT * 2, prefix='      ')
-    write_html_links_index(out_dir, links, finished=finished)
+    write_html_links_index(links, out_dir=out_dir, finished=finished)
     timer.end()
     log_indexing_finished(out_dir, 'index.html')
 
@@ -87,7 +89,7 @@ def load_links_index(out_dir: str=OUTPUT_DIR, import_path: Optional[str]=None) -
 
 
 @enforce_types
-def write_json_links_index(out_dir: str, links: List[Link]) -> None:
+def write_json_links_index(links: List[Link], out_dir: str=OUTPUT_DIR) -> None:
     """write the json link index to a given path"""
 
     assert isinstance(links, List), 'Links must be a list, not a generator.'
@@ -199,7 +201,6 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None:
     successful = link.num_outputs
 
     # Patch JSON index
-    changed = False
     json_file_links = parse_json_links_index(out_dir)
     patched_links = []
     for saved_link in json_file_links:
@@ -212,7 +213,7 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None:
         else:
             patched_links.append(saved_link)
     
-    write_json_links_index(out_dir, patched_links)
+    write_json_links_index(patched_links, out_dir=out_dir)
 
     # Patch HTML index
     html_path = os.path.join(out_dir, 'index.html')
@@ -231,27 +232,27 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None:
 ### Individual link index
 
 @enforce_types
-def write_link_index(out_dir: str, link: Link) -> None:
+def write_link_index(link: Link, link_dir: Optional[str]=None) -> None:
-    write_json_link_index(out_dir, link)
+    link_dir = link_dir or link.link_dir
-    write_html_link_index(out_dir, link)
+
+    write_json_link_index(link, link_dir)
+    write_html_link_index(link, link_dir)
 
 
 @enforce_types
-def write_json_link_index(out_dir: str, link: Link) -> None:
+def write_json_link_index(link: Link, link_dir: Optional[str]=None) -> None:
     """write a json file with some info about the link"""
     
-    path = os.path.join(out_dir, 'index.json')
+    link_dir = link_dir or link.link_dir
-
+    path = os.path.join(link_dir, 'index.json')
-    with open(path, 'w', encoding='utf-8') as f:
-        json.dump(link._asdict(), f, indent=4, cls=ExtendedEncoder)
 
     chmod_file(path)
 
 
 @enforce_types
-def parse_json_link_index(out_dir: str) -> Optional[Link]:
+def parse_json_link_index(link_dir: str) -> Optional[Link]:
     """load the json link index from a given directory"""
-    existing_index = os.path.join(out_dir, 'index.json')
+    existing_index = os.path.join(link_dir, 'index.json')
     if os.path.exists(existing_index):
         with open(existing_index, 'r', encoding='utf-8') as f:
             link_json = json.load(f)
@@ -260,18 +261,21 @@ def parse_json_link_index(out_dir: str) -> Optional[Link]:
 
 
 @enforce_types
-def load_json_link_index(out_dir: str, link: Link) -> Link:
+def load_json_link_index(link: Link, link_dir: Optional[str]=None) -> Link:
     """check for an existing link archive in the given directory, 
        and load+merge it into the given link dict
     """
-    existing_link = parse_json_link_index(out_dir)
+    link_dir = link_dir or link.link_dir
+    existing_link = parse_json_link_index(link_dir)
     if existing_link:
         return merge_links(existing_link, link)
     return link
 
 
 @enforce_types
-def write_html_link_index(out_dir: str, link: Link) -> None:
+def write_html_link_index(link: Link, link_dir: Optional[str]=None) -> None:
+    link_dir = link_dir or link.link_dir
+
     with open(os.path.join(TEMPLATES_DIR, 'link_index.html'), 'r', encoding='utf-8') as f:
         link_html = f.read()
 

archivebox/logs.py

@@ -6,7 +6,7 @@ from dataclasses import dataclass
 from typing import Optional
 
 from .schema import Link, ArchiveResult
-from .config import ANSI, REPO_DIR, OUTPUT_DIR
+from .config import ANSI, OUTPUT_DIR
 
 
 @dataclass
@@ -17,14 +17,14 @@ class RuntimeStats:
     succeeded: int = 0
     failed: int = 0
 
-    parse_start_ts: datetime = None
+    parse_start_ts: Optional[datetime] = None
-    parse_end_ts: datetime = None
+    parse_end_ts: Optional[datetime] = None
 
-    index_start_ts: datetime = None
+    index_start_ts: Optional[datetime] = None
-    index_end_ts: datetime = None
+    index_end_ts: Optional[datetime] = None
 
-    archiving_start_ts: datetime = None
+    archiving_start_ts: Optional[datetime] = None
-    archiving_end_ts: datetime = None
+    archiving_end_ts: Optional[datetime] = None
 
 # globals are bad, mmkay
 _LAST_RUN_STATS = RuntimeStats()
@@ -131,7 +131,7 @@ def log_archiving_finished(num_links: int):
     print('        {}/index.html'.format(OUTPUT_DIR))
 
 
-def log_link_archiving_started(link_dir: str, link: Link, is_new: bool):
+def log_link_archiving_started(link: Link, link_dir: str, is_new: bool):
     # [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford"
     #     http://www.benstopford.com/2015/02/14/log-structured-merge-trees/
     #     > output/archive/1478739709
@@ -149,7 +149,7 @@ def log_link_archiving_started(link_dir: str, link: Link, is_new: bool):
         pretty_path(link_dir),
     ))
 
-def log_link_archiving_finished(link_dir: str, link: Link, is_new: bool, stats: dict):
+def log_link_archiving_finished(link: Link, link_dir: str, is_new: bool, stats: dict):
     total = sum(stats.values())
 
     if stats['failed'] > 0 :

archivebox/util.py

@@ -1,11 +1,12 @@
 import os
 import re
 import sys
+import json
 import time
 import shutil
 
 from json import JSONEncoder
-from typing import List, Optional, Any
+from typing import List, Optional, Any, Union
 from inspect import signature, _empty
 from functools import wraps
 from hashlib import sha256