mirror of
https://github.com/ArchiveBox/ArchiveBox
synced 2024-11-25 13:40:20 +00:00
new methods for detecting valid/invalid data dirs on init
This commit is contained in:
Nick Sweeting
parent
ae782a1a0c
commit
56d0b2c088
2 changed files with 355 additions and 54 deletions
|
|
@ -7,11 +7,21 @@ __description__ = 'List all the URLs currently in the archive.'
|
|||
import sys
|
||||
import argparse
|
||||
|
||||
|
||||
from ..legacy.util import reject_stdin, to_json, to_csv
|
||||
from ..legacy.config import check_data_folder
|
||||
from ..legacy.main import list_archive_data
|
||||
|
||||
from ..legacy.util import SmartFormatter, reject_stdin, to_json, to_csv
|
||||
from ..legacy.config import check_data_folder, OUTPUT_DIR
|
||||
from ..legacy.main import (
|
||||
list_archive_data,
|
||||
get_indexed_folders,
|
||||
get_archived_folders,
|
||||
get_unarchived_folders,
|
||||
get_present_folders,
|
||||
get_valid_folders,
|
||||
get_invalid_folders,
|
||||
get_duplicate_folders,
|
||||
get_orphaned_folders,
|
||||
get_corrupted_folders,
|
||||
get_unrecognized_folders,
|
||||
)
|
||||
|
||||
def main(args=None):
|
||||
check_data_folder()
|
||||
|
|
@ -22,6 +32,7 @@ def main(args=None):
|
|||
prog=__command__,
|
||||
description=__description__,
|
||||
add_help=True,
|
||||
formatter_class=SmartFormatter,
|
||||
)
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
group.add_argument(
|
||||
|
|
@ -44,15 +55,36 @@ def main(args=None):
|
|||
parser.add_argument(
|
||||
'--before', #'-b',
|
||||
type=float,
|
||||
help="List only URLs bookmarked before the given timestamp.",
|
||||
help="List only links bookmarked before the given timestamp.",
|
||||
default=None,
|
||||
)
|
||||
parser.add_argument(
|
||||
'--after', #'-a',
|
||||
type=float,
|
||||
help="List only URLs bookmarked after the given timestamp.",
|
||||
help="List only links bookmarked after the given timestamp.",
|
||||
default=None,
|
||||
)
|
||||
parser.add_argument(
|
||||
'--status',
|
||||
type=str,
|
||||
choices=('indexed', 'archived', 'unarchived', 'present', 'valid', 'invalid', 'duplicate', 'orphaned', 'corrupted', 'unrecognized'),
|
||||
default='indexed',
|
||||
help=(
|
||||
'List only links or data directories that have the given status\n'
|
||||
f' indexed {get_indexed_folders.__doc__} (the default)\n'
|
||||
f' archived {get_archived_folders.__doc__}\n'
|
||||
f' unarchived {get_unarchived_folders.__doc__}\n'
|
||||
'\n'
|
||||
f' present {get_present_folders.__doc__}\n'
|
||||
f' valid {get_valid_folders.__doc__}\n'
|
||||
f' invalid {get_invalid_folders.__doc__}\n'
|
||||
'\n'
|
||||
f' duplicate {get_duplicate_folders.__doc__}\n'
|
||||
f' orphaned {get_orphaned_folders.__doc__}\n'
|
||||
f' corrupted {get_corrupted_folders.__doc__}\n'
|
||||
f' unrecognized {get_unrecognized_folders.__doc__}\n'
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
'--filter-type',
|
||||
type=str,
|
||||
|
|
@ -76,17 +108,40 @@ def main(args=None):
|
|||
before=command.before,
|
||||
after=command.after,
|
||||
)
|
||||
|
||||
if command.sort:
|
||||
links = sorted(links, key=lambda link: getattr(link, command.sort))
|
||||
|
||||
if command.status == 'indexed':
|
||||
folders = get_indexed_folders(links, out_dir=OUTPUT_DIR)
|
||||
elif command.status == 'archived':
|
||||
folders = get_archived_folders(links, out_dir=OUTPUT_DIR)
|
||||
elif command.status == 'unarchived':
|
||||
folders = get_unarchived_folders(links, out_dir=OUTPUT_DIR)
|
||||
|
||||
elif command.status == 'present':
|
||||
folders = get_present_folders(links, out_dir=OUTPUT_DIR)
|
||||
elif command.status == 'valid':
|
||||
folders = get_valid_folders(links, out_dir=OUTPUT_DIR)
|
||||
elif command.status == 'invalid':
|
||||
folders = get_invalid_folders(links, out_dir=OUTPUT_DIR)
|
||||
|
||||
elif command.status == 'duplicate':
|
||||
folders = get_duplicate_folders(links, out_dir=OUTPUT_DIR)
|
||||
elif command.status == 'orphaned':
|
||||
folders = get_orphaned_folders(links, out_dir=OUTPUT_DIR)
|
||||
elif command.status == 'corrupted':
|
||||
folders = get_corrupted_folders(links, out_dir=OUTPUT_DIR)
|
||||
elif command.status == 'unrecognized':
|
||||
folders = get_unrecognized_folders(links, out_dir=OUTPUT_DIR)
|
||||
|
||||
if command.csv:
|
||||
print(to_csv(links, csv_cols=command.csv.split(','), header=True))
|
||||
print(to_csv(folders.values(), csv_cols=command.csv.split(','), header=True))
|
||||
elif command.json:
|
||||
print(to_json(list(links), indent=4, sort_keys=True))
|
||||
print(to_json(folders.values(), indent=4, sort_keys=True))
|
||||
else:
|
||||
print('\n'.join(link.url for link in links))
|
||||
|
||||
print('\n'.join(f'{folder} {link}' for folder, link in folders.items()))
|
||||
raise SystemExit(not folders)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@ import os
|
|||
import re
|
||||
import shutil
|
||||
|
||||
from typing import List, Optional, Iterable
|
||||
from typing import Dict, List, Optional, Iterable
|
||||
from itertools import chain
|
||||
|
||||
from .schema import Link
|
||||
from .util import (
|
||||
|
|
@ -17,8 +18,13 @@ from .index import (
|
|||
import_new_links,
|
||||
write_main_index,
|
||||
)
|
||||
from .storage.json import parse_json_main_index, parse_json_links_details
|
||||
from .storage.sql import parse_sql_main_index
|
||||
from .storage.json import (
|
||||
parse_json_main_index,
|
||||
parse_json_link_details,
|
||||
parse_json_links_details,
|
||||
)
|
||||
from .storage.sql import parse_sql_main_index, get_admins
|
||||
from .storage.html import parse_html_main_index
|
||||
from .archive_methods import archive_link
|
||||
from .config import (
|
||||
stderr,
|
||||
|
|
@ -164,11 +170,39 @@ def init():
|
|||
orphaned_data_dir_links = {
|
||||
link.url: link
|
||||
for link in parse_json_links_details(OUTPUT_DIR)
|
||||
if link.url not in all_links
|
||||
}
|
||||
if orphaned_data_dir_links:
|
||||
all_links.update(orphaned_data_dir_links)
|
||||
print(' {lightyellow}√ Added {} orphaned links from existing archive directories...{reset}'.format(len(orphaned_data_dir_links), **ANSI))
|
||||
orphan_new_links = {
|
||||
url: link
|
||||
for url, link in orphaned_data_dir_links.items()
|
||||
if url not in all_links
|
||||
}
|
||||
orphan_duplicates = {
|
||||
url: link
|
||||
for url, link in orphaned_data_dir_links.items()
|
||||
if url in all_links
|
||||
}
|
||||
if orphan_new_links:
|
||||
all_links.update(orphan_new_links)
|
||||
print(' {lightyellow}√ Added {} orphaned links from existing archive directories...{reset}'.format(len(orphan_new_links), **ANSI))
|
||||
if orphan_duplicates:
|
||||
print(' {lightyellow}! Skipped adding {} orphaned link data directories that would have overwritten existing data.{reset}'.format(len(orphan_duplicates), **ANSI))
|
||||
|
||||
orphaned_data_dirs = {folder for folder in orphan_duplicates.keys()}
|
||||
invalid_folders = {
|
||||
folder: link
|
||||
for folder, link in get_invalid_folders(all_links.values(), out_dir=OUTPUT_DIR).items()
|
||||
if folder not in orphaned_data_dirs
|
||||
}
|
||||
if invalid_folders:
|
||||
print(' {lightyellow}! Skipped adding {} corrupted/unrecognized link data directories that could not be read.{reset}'.format(len(orphan_duplicates), **ANSI))
|
||||
|
||||
if orphan_duplicates or invalid_folders:
|
||||
print(' For more information about the link data directories that were skipped, run:')
|
||||
print(' archivebox info')
|
||||
print(' archivebox list --status=invalid')
|
||||
print(' archivebox list --status=orphaned')
|
||||
print(' archivebox list --status=duplicate')
|
||||
|
||||
|
||||
write_main_index(list(all_links.values()), out_dir=OUTPUT_DIR)
|
||||
|
||||
|
|
@ -190,55 +224,88 @@ def init():
|
|||
|
||||
@enforce_types
|
||||
def info():
|
||||
all_links = load_main_index(out_dir=OUTPUT_DIR)
|
||||
|
||||
print('{green}[*] Scanning archive collection main index with {} links:{reset}'.format(len(all_links), **ANSI))
|
||||
print(f' {OUTPUT_DIR}')
|
||||
num_bytes, num_dirs, num_files = get_dir_size(OUTPUT_DIR, recursive=False)
|
||||
print('{green}[*] Scanning archive collection main index...{reset}'.format(**ANSI))
|
||||
print(f' {OUTPUT_DIR}/*')
|
||||
num_bytes, num_dirs, num_files = get_dir_size(OUTPUT_DIR, recursive=False, pattern='index.')
|
||||
size = human_readable_size(num_bytes)
|
||||
print(f' > Index Size: {size} across {num_files} files')
|
||||
print(f' Size: {size} across {num_files} files')
|
||||
print()
|
||||
|
||||
setup_django()
|
||||
from django.contrib.auth.models import User
|
||||
from core.models import Page
|
||||
links = load_main_index(out_dir=OUTPUT_DIR)
|
||||
num_json_links = len(links)
|
||||
num_sql_links = sum(1 for link in parse_sql_main_index(out_dir=OUTPUT_DIR))
|
||||
num_html_links = sum(1 for url in parse_html_main_index(out_dir=OUTPUT_DIR))
|
||||
num_link_details = sum(1 for link in parse_json_links_details(out_dir=OUTPUT_DIR))
|
||||
users = get_admins().values_list('username', flat=True)
|
||||
print(f' > JSON Main Index: {num_json_links} links'.ljust(36), f'(found in {JSON_INDEX_FILENAME})')
|
||||
print(f' > SQL Main Index: {num_sql_links} links'.ljust(36), f'(found in {SQL_INDEX_FILENAME})')
|
||||
print(f' > HTML Main Index: {num_html_links} links'.ljust(36), f'(found in {HTML_INDEX_FILENAME})')
|
||||
print(f' > JSON Link Details: {num_link_details} links'.ljust(36), f'(found in {ARCHIVE_DIR_NAME}/*/index.json)')
|
||||
|
||||
users = User.objects.all()
|
||||
num_pages = Page.objects.count()
|
||||
print(f' > Admin: {len(users)} users {", ".join(users)}'.ljust(36), f'(found in {SQL_INDEX_FILENAME})')
|
||||
|
||||
print(f' > {len(users)} admin users:', ', '.join(u.username for u in users))
|
||||
print(f' > {num_pages} pages in SQL database {SQL_INDEX_FILENAME}')
|
||||
print(f' > {len(all_links)} pages in JSON database {JSON_INDEX_FILENAME}')
|
||||
print()
|
||||
if num_html_links != len(links) or num_sql_links != len(links):
|
||||
print()
|
||||
print(' {lightred}Hint:{reset} You can fix index count differences automatically by running:'.format(**ANSI))
|
||||
print(' archivebox init')
|
||||
|
||||
if not users:
|
||||
print()
|
||||
print(' {lightred}Hint:{reset} You can create an admin user by running:'.format(**ANSI))
|
||||
print(' archivebox manage createsuperuser')
|
||||
|
||||
print('{green}[*] Scanning archive collection data directory with {} entries:{reset}'.format(len(all_links), **ANSI))
|
||||
print(f' {ARCHIVE_DIR}')
|
||||
print()
|
||||
print('{green}[*] Scanning archive collection link data directories...{reset}'.format(**ANSI))
|
||||
print(f' {ARCHIVE_DIR}/*')
|
||||
|
||||
num_bytes, num_dirs, num_files = get_dir_size(ARCHIVE_DIR)
|
||||
size = human_readable_size(num_bytes)
|
||||
print(f' > Total Size: {size} across {num_files} files in {num_dirs} directories')
|
||||
print(f' Size: {size} across {num_files} files in {num_dirs} directories')
|
||||
print()
|
||||
|
||||
link_data_dirs = {link.link_dir for link in all_links}
|
||||
valid_archive_dirs = set()
|
||||
num_invalid = 0
|
||||
for entry in os.scandir(ARCHIVE_DIR):
|
||||
if entry.is_dir(follow_symlinks=True):
|
||||
if os.path.exists(os.path.join(entry.path, 'index.json')):
|
||||
valid_archive_dirs.add(entry.path)
|
||||
else:
|
||||
num_invalid += 1
|
||||
|
||||
print(f' > {len(valid_archive_dirs)} valid archive data directories (valid directories matched to links in the index)')
|
||||
|
||||
num_unarchived = sum(1 for link in all_links if link.link_dir not in valid_archive_dirs)
|
||||
print(f' > {num_unarchived} missing data directories (directories missing for links in the index)')
|
||||
|
||||
print(f' > {num_invalid} invalid data directories (directories present that don\'t contain an index file)')
|
||||
|
||||
num_orphaned = sum(1 for data_dir in valid_archive_dirs if data_dir not in link_data_dirs)
|
||||
print(f' > {num_orphaned} orphaned data directories (directories present for links that don\'t exist in the index)')
|
||||
num_indexed = len(get_indexed_folders(links, out_dir=OUTPUT_DIR))
|
||||
num_archived = len(get_archived_folders(links, out_dir=OUTPUT_DIR))
|
||||
num_unarchived = len(get_unarchived_folders(links, out_dir=OUTPUT_DIR))
|
||||
print(f' > indexed: {num_indexed}'.ljust(36), f'({get_indexed_folders.__doc__})')
|
||||
print(f' > archived: {num_archived}'.ljust(36), f'({get_archived_folders.__doc__})')
|
||||
print(f' > unarchived: {num_unarchived}'.ljust(36), f'({get_unarchived_folders.__doc__})')
|
||||
|
||||
num_present = len(get_present_folders(links, out_dir=OUTPUT_DIR))
|
||||
num_valid = len(get_valid_folders(links, out_dir=OUTPUT_DIR))
|
||||
print()
|
||||
print(f' > present: {num_present}'.ljust(36), f'({get_present_folders.__doc__})')
|
||||
print(f' > valid: {num_valid}'.ljust(36), f'({get_valid_folders.__doc__})')
|
||||
|
||||
duplicate = get_duplicate_folders(links, out_dir=OUTPUT_DIR)
|
||||
orphaned = get_orphaned_folders(links, out_dir=OUTPUT_DIR)
|
||||
corrupted = get_corrupted_folders(links, out_dir=OUTPUT_DIR)
|
||||
unrecognized = get_unrecognized_folders(links, out_dir=OUTPUT_DIR)
|
||||
num_invalid = len({**duplicate, **orphaned, **corrupted, **unrecognized})
|
||||
print(f' > invalid: {num_invalid}'.ljust(36), f'({get_invalid_folders.__doc__})')
|
||||
print(f' > duplicate: {len(duplicate)}'.ljust(36), f'({get_duplicate_folders.__doc__})')
|
||||
print(f' > orphaned: {len(orphaned)}'.ljust(36), f'({get_orphaned_folders.__doc__})')
|
||||
print(f' > corrupted: {len(corrupted)}'.ljust(36), f'({get_corrupted_folders.__doc__})')
|
||||
print(f' > unrecognized: {len(unrecognized)}'.ljust(36), f'({get_unrecognized_folders.__doc__})')
|
||||
|
||||
if num_indexed:
|
||||
print()
|
||||
print(' {lightred}Hint:{reset} You can list link data directories by status like so:'.format(**ANSI))
|
||||
print(' archivebox list --status=<status> (e.g. indexed, corrupted, archived, etc.)')
|
||||
|
||||
if orphaned:
|
||||
print()
|
||||
print(' {lightred}Hint:{reset} To automatically import orphaned data directories into the main index, run:'.format(**ANSI))
|
||||
print(' archivebox init')
|
||||
|
||||
if num_invalid:
|
||||
print()
|
||||
print(' {lightred}Hint:{reset} You may need to manually remove or fix some invalid data directories, afterwards make sure to run:'.format(**ANSI))
|
||||
print(' archivebox init')
|
||||
|
||||
print()
|
||||
|
||||
|
||||
|
||||
@enforce_types
|
||||
def update_archive_data(import_path: Optional[str]=None, resume: Optional[float]=None, only_new: bool=False) -> List[Link]:
|
||||
|
|
@ -367,3 +434,182 @@ def remove_archive_links(filter_patterns: List[str], filter_type: str='exact',
|
|||
log_removal_finished(len(all_links), len(to_keep))
|
||||
|
||||
return to_keep
|
||||
|
||||
|
||||
|
||||
def get_indexed_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
|
||||
"""indexed links without checking archive status or data directory validity"""
|
||||
return {
|
||||
link.link_dir: link
|
||||
for link in links
|
||||
}
|
||||
|
||||
def get_archived_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
|
||||
"""indexed links that are archived with a valid data directory"""
|
||||
return {
|
||||
link.link_dir: link
|
||||
for link in filter(is_archived, links)
|
||||
}
|
||||
|
||||
def get_unarchived_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
|
||||
"""indexed links that are unarchived with no data directory or an empty data directory"""
|
||||
return {
|
||||
link.link_dir: link
|
||||
for link in filter(is_unarchived, links)
|
||||
}
|
||||
|
||||
def get_present_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
|
||||
"""dirs that are expected to exist based on the main index"""
|
||||
all_folders = {}
|
||||
|
||||
for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME)):
|
||||
if entry.is_dir(follow_symlinks=True):
|
||||
link = None
|
||||
try:
|
||||
link = parse_json_link_details(entry.path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
all_folders[entry.path] = link
|
||||
|
||||
return all_folders
|
||||
|
||||
def get_valid_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
|
||||
"""dirs with a valid index matched to the main index and archived content"""
|
||||
return {
|
||||
link.link_dir: link
|
||||
for link in filter(is_valid, links)
|
||||
}
|
||||
|
||||
def get_invalid_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
|
||||
"""dirs that are invalid for any reason: corrupted/duplicate/orphaned/unrecognized"""
|
||||
duplicate = get_duplicate_folders(links, out_dir=OUTPUT_DIR)
|
||||
orphaned = get_orphaned_folders(links, out_dir=OUTPUT_DIR)
|
||||
corrupted = get_corrupted_folders(links, out_dir=OUTPUT_DIR)
|
||||
unrecognized = get_unrecognized_folders(links, out_dir=OUTPUT_DIR)
|
||||
return {**duplicate, **orphaned, **corrupted, **unrecognized}
|
||||
|
||||
|
||||
def get_duplicate_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
|
||||
"""dirs that conflict with other directories that have the same link URL or timestamp"""
|
||||
links = list(links)
|
||||
by_url = {link.url: 0 for link in links}
|
||||
by_timestamp = {link.timestamp: 0 for link in links}
|
||||
|
||||
duplicate_folders = {}
|
||||
|
||||
indexed_folders = {link.link_dir for link in links}
|
||||
data_folders = (
|
||||
entry.path
|
||||
for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME))
|
||||
if entry.is_dir(follow_symlinks=True) and entry.path not in indexed_folders
|
||||
)
|
||||
|
||||
for path in chain(sorted(indexed_folders), sorted(data_folders)):
|
||||
link = None
|
||||
try:
|
||||
link = parse_json_link_details(path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if link:
|
||||
# link folder has same timestamp as different link folder
|
||||
by_timestamp[link.timestamp] = by_timestamp.get(link.timestamp, 0) + 1
|
||||
if by_timestamp[link.timestamp] > 1:
|
||||
duplicate_folders[path] = link
|
||||
|
||||
# link folder has same url as different link folder
|
||||
by_url[link.url] = by_url.get(link.url, 0) + 1
|
||||
if by_url[link.url] > 1:
|
||||
duplicate_folders[path] = link
|
||||
|
||||
return duplicate_folders
|
||||
|
||||
def get_orphaned_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
|
||||
"""dirs that contain a valid index but aren't listed in the main index"""
|
||||
links = list(links)
|
||||
indexed_folders = {link.link_dir: link for link in links}
|
||||
orphaned_folders = {}
|
||||
|
||||
for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME)):
|
||||
if entry.is_dir(follow_symlinks=True):
|
||||
index_exists = os.path.exists(os.path.join(entry.path, 'index.json'))
|
||||
link = None
|
||||
try:
|
||||
link = parse_json_link_details(entry.path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if index_exists and entry.path not in indexed_folders:
|
||||
# folder is a valid link data dir with index details, but it's not in the main index
|
||||
orphaned_folders[entry.path] = link
|
||||
|
||||
return orphaned_folders
|
||||
|
||||
def get_corrupted_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
|
||||
"""dirs that don't contain a valid index and aren't listed in the main index"""
|
||||
return {
|
||||
link.link_dir: link
|
||||
for link in filter(is_corrupt, links)
|
||||
}
|
||||
|
||||
def get_unrecognized_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
|
||||
"""dirs that don't contain recognizable archive data and aren't listed in the main index"""
|
||||
by_timestamp = {link.timestamp: 0 for link in links}
|
||||
unrecognized_folders: Dict[str, Optional[Link]] = {}
|
||||
|
||||
for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME)):
|
||||
if entry.is_dir(follow_symlinks=True):
|
||||
index_exists = os.path.exists(os.path.join(entry.path, 'index.json'))
|
||||
link = None
|
||||
try:
|
||||
link = parse_json_link_details(entry.path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if index_exists and link is None:
|
||||
# index exists but it's corrupted or unparseable
|
||||
unrecognized_folders[entry.path] = link
|
||||
|
||||
elif not index_exists:
|
||||
# link details index doesn't exist and the folder isn't in the main index
|
||||
timestamp = entry.path.rsplit('/', 1)[-1]
|
||||
if timestamp not in by_timestamp:
|
||||
unrecognized_folders[entry.path] = link
|
||||
|
||||
return unrecognized_folders
|
||||
|
||||
|
||||
def is_valid(link: Link) -> bool:
|
||||
dir_exists = os.path.exists(link.link_dir)
|
||||
index_exists = os.path.exists(os.path.join(link.link_dir, 'index.json'))
|
||||
if not dir_exists:
|
||||
# unarchived links are not included in the valid list
|
||||
return False
|
||||
if dir_exists and not index_exists:
|
||||
return False
|
||||
if dir_exists and index_exists:
|
||||
try:
|
||||
parsed_link = parse_json_link_details(link.link_dir)
|
||||
return link.url == parsed_link.url
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
def is_corrupt(link: Link) -> bool:
|
||||
if not os.path.exists(link.link_dir):
|
||||
# unarchived links are not considered corrupt
|
||||
return False
|
||||
|
||||
if is_valid(link):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def is_archived(link: Link) -> bool:
|
||||
return is_valid(link) and link.is_archived
|
||||
|
||||
def is_unarchived(link: Link) -> bool:
|
||||
if not os.path.exists(link.link_dir):
|
||||
return True
|
||||
return not link.is_archived
|
||||
|
|
|
|||
Loading…
Reference in a new issue