Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-10 14:44:18 +00:00
Code Issues Projects Releases Packages Wiki Activity

feat: Update extractors and add command to use sql index as source of truth

Browse source
This commit is contained in:
Cristian 2020-08-18 12:24:43 -05:00 committed by Cristian Vargas
parent e9caee6b10
commit 31343c1367
3 changed files with 3 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
archivebox/extractors/__init__.py
View file

@ -113,8 +113,6 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
pass
write_link_details(link, out_dir=out_dir, skip_sql_index=skip_index)
if not skip_index:
patch_main_index(link)
# # If any changes were made, update the main links index json and html
# was_changed = stats['succeeded'] or stats['failed']

4
archivebox/index/__init__.py
View file

@ -236,7 +236,7 @@ def timed_index_update(out_path: str):
@enforce_types
def write_main_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=False, write_static: bool=False) -> None:
def write_main_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=False) -> None:
"""create index.html file for a given list of links"""
log_indexing_process_started(len(links))
@ -246,7 +246,7 @@ def write_main_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=
write_sql_main_index(links, out_dir=out_dir)
os.chmod(os.path.join(out_dir, SQL_INDEX_FILENAME), int(OUTPUT_PERMISSIONS, base=8)) # set here because we don't write it with atomic writes
if write_static:
if finished:
with timed_index_update(os.path.join(out_dir, JSON_INDEX_FILENAME)):
write_json_main_index(links, out_dir=out_dir)

2
archivebox/main.py
View file

@ -368,7 +368,7 @@ def init(force: bool=False, out_dir: str=OUTPUT_DIR) -> None:
print(' archivebox list --status=invalid')
write_main_index(list(all_links.values()), out_dir=out_dir, write_static=True)
write_main_index(list(all_links.values()), out_dir=out_dir)
print('\n{green}------------------------------------------------------------------{reset}'.format(**ANSI))
if existing_index: