Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-22 12:13:05 +00:00
Code Issues Projects Releases Packages Wiki Activity

add timezone support, tons of CSS and layout improvements, more detailed snapshot admin form info, ability to sort by recently updated, better grid view styling, better table layouts, better dark mode support

Browse source
This commit is contained in:
Nick Sweeting 2021-04-10 04:19:30 -04:00
parent cf7d7e4990
commit a9986f1f05
28 changed files with 681 additions and 549 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
archivebox/config.py
View file

@ -34,7 +34,7 @@ import django
from hashlib import md5 from hashlib import md5
from pathlib import Path from pathlib import Path
from datetime import datetime from datetime import datetime, timezone
from typing import Optional, Type, Tuple, Dict, Union, List from typing import Optional, Type, Tuple, Dict, Union, List
from subprocess import run, PIPE, DEVNULL from subprocess import run, PIPE, DEVNULL
from configparser import ConfigParser from configparser import ConfigParser
@ -80,7 +80,8 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
'PUBLIC_ADD_VIEW': {'type': bool, 'default': False}, 'PUBLIC_ADD_VIEW': {'type': bool, 'default': False},
'FOOTER_INFO': {'type': str, 'default': 'Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.'}, 'FOOTER_INFO': {'type': str, 'default': 'Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.'},
'SNAPSHOTS_PER_PAGE': {'type': int, 'default': 40}, 'SNAPSHOTS_PER_PAGE': {'type': int, 'default': 40},
'CUSTOM_TEMPLATES_DIR': {'type': str, 'default': None} 'CUSTOM_TEMPLATES_DIR': {'type': str, 'default': None},
'TIME_ZONE': {'type': str, 'default': 'UTC'},
}, },
'ARCHIVE_METHOD_TOGGLES': { 'ARCHIVE_METHOD_TOGGLES': {
@ -1105,7 +1106,7 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG,
# log startup message to the error log # log startup message to the error log
with open(settings.ERROR_LOG, "a+", encoding='utf-8') as f: with open(settings.ERROR_LOG, "a+", encoding='utf-8') as f:
command = ' '.join(sys.argv) command = ' '.join(sys.argv)
ts = datetime.now().strftime('%Y-%m-%d__%H:%M:%S') ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S')
f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n") f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n")

1
archivebox/core/admin.py
View file

@ -3,6 +3,7 @@ __package__ = 'archivebox.core'
from io import StringIO from io import StringIO
from pathlib import Path from pathlib import Path
from contextlib import redirect_stdout from contextlib import redirect_stdout
from datetime import datetime, timezone
from django.contrib import admin from django.contrib import admin
from django.urls import path from django.urls import path

17
archivebox/core/settings.py
View file

@ -19,9 +19,9 @@ from ..config import (
SQL_INDEX_FILENAME, SQL_INDEX_FILENAME,
OUTPUT_DIR, OUTPUT_DIR,
LOGS_DIR, LOGS_DIR,
TIME_ZONE,
) )
IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3] IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3] IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
@ -154,6 +154,7 @@ DATABASES = {
'timeout': 60, 'timeout': 60,
'check_same_thread': False, 'check_same_thread': False,
}, },
'TIME_ZONE': 'UTC',
# DB setup is sometimes modified at runtime by setup_django() in config.py # DB setup is sometimes modified at runtime by setup_django() in config.py
} }
} }
@ -182,6 +183,7 @@ ALLOWED_HOSTS = ALLOWED_HOSTS.split(',')
SECURE_BROWSER_XSS_FILTER = True SECURE_BROWSER_XSS_FILTER = True
SECURE_CONTENT_TYPE_NOSNIFF = True SECURE_CONTENT_TYPE_NOSNIFF = True
SECURE_REFERRER_POLICY = 'strict-origin-when-cross-origin'
CSRF_COOKIE_SECURE = False CSRF_COOKIE_SECURE = False
SESSION_COOKIE_SECURE = False SESSION_COOKIE_SECURE = False
@ -217,14 +219,17 @@ if IS_SHELL:
################################################################################ ################################################################################
LANGUAGE_CODE = 'en-us' LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC' USE_I18N = True
USE_I18N = False USE_L10N = True
USE_L10N = False USE_TZ = True
USE_TZ = False
DATETIME_FORMAT = 'Y-m-d g:iA' DATETIME_FORMAT = 'Y-m-d g:iA'
SHORT_DATETIME_FORMAT = 'Y-m-d h:iA' SHORT_DATETIME_FORMAT = 'Y-m-d h:iA'
from django.conf.locale.en import formats as en_formats
en_formats.DATETIME_FORMAT = DATETIME_FORMAT
en_formats.SHORT_DATETIME_FORMAT = SHORT_DATETIME_FORMAT
################################################################################ ################################################################################
### Logging Settings ### Logging Settings

13
archivebox/core/templatetags/core_tags.py
View file

@ -1,22 +1,15 @@
from django import template from django import template
from django.urls import reverse
from django.contrib.admin.templatetags.base import InclusionAdminNode from django.contrib.admin.templatetags.base import InclusionAdminNode
from django.templatetags.static import static
from typing import Union from typing import Union
from core.models import ArchiveResult
register = template.Library() register = template.Library()
@register.simple_tag @register.filter(name='split')
def snapshot_image(snapshot): def split(value, separator: str=','):
result = ArchiveResult.objects.filter(snapshot=snapshot, extractor='screenshot', status='succeeded').first() return (value or '').split(separator)
if result:
return reverse('Snapshot', args=[f'{str(snapshot.timestamp)}/{result.output}'])
return static('archive.png')
@register.filter @register.filter
def file_size(num_bytes: Union[int, float]) -> str: def file_size(num_bytes: Union[int, float]) -> str:

4
archivebox/extractors/__init__.py
View file

@ -4,7 +4,7 @@ import os
from pathlib import Path from pathlib import Path
from typing import Optional, List, Iterable, Union from typing import Optional, List, Iterable, Union
from datetime import datetime from datetime import datetime, timezone
from django.db.models import QuerySet from django.db.models import QuerySet
from ..index.schema import Link from ..index.schema import Link
@ -94,7 +94,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
link = load_link_details(link, out_dir=out_dir) link = load_link_details(link, out_dir=out_dir)
write_link_details(link, out_dir=out_dir, skip_sql_index=False) write_link_details(link, out_dir=out_dir, skip_sql_index=False)
log_link_archiving_started(link, out_dir, is_new) log_link_archiving_started(link, out_dir, is_new)
link = link.overwrite(updated=datetime.now()) link = link.overwrite(updated=datetime.now(timezone.utc))
stats = {'skipped': 0, 'succeeded': 0, 'failed': 0} stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
for method_name, should_run, method_function in ARCHIVE_METHODS: for method_name, should_run, method_function in ARCHIVE_METHODS:

1
archivebox/extractors/readability.py
View file

@ -92,6 +92,7 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
result = run(cmd, cwd=out_dir, timeout=timeout) result = run(cmd, cwd=out_dir, timeout=timeout)
try: try:
result_json = json.loads(result.stdout) result_json = json.loads(result.stdout)
assert result_json and 'content' in result_json
except json.JSONDecodeError: except json.JSONDecodeError:
raise ArchiveError('Readability was not able to archive the page', result.stdout + result.stderr) raise ArchiveError('Readability was not able to archive the page', result.stdout + result.stderr)

4
archivebox/extractors/wget.py
View file

@ -4,7 +4,7 @@ import re
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Optional
from datetime import datetime from datetime import datetime, timezone
from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from ..system import run, chmod_file from ..system import run, chmod_file
@ -51,7 +51,7 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
if SAVE_WARC: if SAVE_WARC:
warc_dir = out_dir / "warc" warc_dir = out_dir / "warc"
warc_dir.mkdir(exist_ok=True) warc_dir.mkdir(exist_ok=True)
warc_path = warc_dir / str(int(datetime.now().timestamp())) warc_path = warc_dir / str(int(datetime.now(timezone.utc).timestamp()))
# WGET CLI Docs: https://www.gnu.org/software/wget/manual/wget.html # WGET CLI Docs: https://www.gnu.org/software/wget/manual/wget.html
output: ArchiveOutput = None output: ArchiveOutput = None

14
archivebox/index/html.py
View file

@ -1,7 +1,7 @@
__package__ = 'archivebox.index' __package__ = 'archivebox.index'
from pathlib import Path from pathlib import Path
from datetime import datetime from datetime import datetime, timezone
from collections import defaultdict from collections import defaultdict
from typing import List, Optional, Iterator, Mapping from typing import List, Optional, Iterator, Mapping
@ -13,7 +13,7 @@ from ..system import atomic_write
from ..logging_util import printable_filesize from ..logging_util import printable_filesize
from ..util import ( from ..util import (
enforce_types, enforce_types,
ts_to_date, ts_to_date_str,
urlencode, urlencode,
htmlencode, htmlencode,
urldecode, urldecode,
@ -62,8 +62,8 @@ def main_index_template(links: List[Link], template: str=MAIN_INDEX_TEMPLATE) ->
'version': VERSION, 'version': VERSION,
'git_sha': VERSION, # not used anymore, but kept for backwards compatibility 'git_sha': VERSION, # not used anymore, but kept for backwards compatibility
'num_links': str(len(links)), 'num_links': str(len(links)),
'date_updated': datetime.now().strftime('%Y-%m-%d'), 'date_updated': datetime.now(timezone.utc).strftime('%Y-%m-%d'),
'time_updated': datetime.now().strftime('%Y-%m-%d %H:%M'), 'time_updated': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M'),
'links': [link._asdict(extended=True) for link in links], 'links': [link._asdict(extended=True) for link in links],
'FOOTER_INFO': FOOTER_INFO, 'FOOTER_INFO': FOOTER_INFO,
}) })
@ -103,7 +103,7 @@ def link_details_template(link: Link) -> str:
'size': printable_filesize(link.archive_size) if link.archive_size else 'pending', 'size': printable_filesize(link.archive_size) if link.archive_size else 'pending',
'status': 'archived' if link.is_archived else 'not yet archived', 'status': 'archived' if link.is_archived else 'not yet archived',
'status_color': 'success' if link.is_archived else 'danger', 'status_color': 'success' if link.is_archived else 'danger',
'oldest_archive_date': ts_to_date(link.oldest_archive_date), 'oldest_archive_date': ts_to_date_str(link.oldest_archive_date),
'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG, 'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
}) })
@ -120,7 +120,7 @@ def snapshot_icons(snapshot) -> str:
def calc_snapshot_icons(): def calc_snapshot_icons():
from core.models import EXTRACTORS from core.models import EXTRACTORS
# start = datetime.now() # start = datetime.now(timezone.utc)
archive_results = snapshot.archiveresult_set.filter(status="succeeded", output__isnull=False) archive_results = snapshot.archiveresult_set.filter(status="succeeded", output__isnull=False)
link = snapshot.as_link() link = snapshot.as_link()
@ -183,7 +183,7 @@ def snapshot_icons(snapshot) -> str:
"archive_org", icons.get("archive_org", "?")) "archive_org", icons.get("archive_org", "?"))
result = format_html('<span class="files-icons" style="font-size: 1.1em; opacity: 0.8; min-width: 240px; display: inline-block">{}<span>', mark_safe(output)) result = format_html('<span class="files-icons" style="font-size: 1.1em; opacity: 0.8; min-width: 240px; display: inline-block">{}<span>', mark_safe(output))
# end = datetime.now() # end = datetime.now(timezone.utc)
# print(((end - start).total_seconds()*1000) // 1, 'ms') # print(((end - start).total_seconds()*1000) // 1, 'ms')
return result return result

4
archivebox/index/json.py
View file

@ -5,7 +5,7 @@ import sys
import json as pyjson import json as pyjson
from pathlib import Path from pathlib import Path
from datetime import datetime from datetime import datetime, timezone
from typing import List, Optional, Iterator, Any, Union from typing import List, Optional, Iterator, Any, Union
from .schema import Link from .schema import Link
@ -44,7 +44,7 @@ def generate_json_index_from_links(links: List[Link], with_headers: bool):
output = { output = {
**MAIN_INDEX_HEADER, **MAIN_INDEX_HEADER,
'num_links': len(links), 'num_links': len(links),
'updated': datetime.now(), 'updated': datetime.now(timezone.utc),
'last_run_cmd': sys.argv, 'last_run_cmd': sys.argv,
'links': links, 'links': links,
} }

17
archivebox/index/schema.py
View file

@ -10,7 +10,7 @@ __package__ = 'archivebox.index'
from pathlib import Path from pathlib import Path
from datetime import datetime, timedelta from datetime import datetime, timezone, timedelta
from typing import List, Dict, Any, Optional, Union from typing import List, Dict, Any, Optional, Union
@ -19,7 +19,7 @@ from dataclasses import dataclass, asdict, field, fields
from django.utils.functional import cached_property from django.utils.functional import cached_property
from ..system import get_dir_size from ..system import get_dir_size
from ..util import ts_to_date_str, parse_date
from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME
class ArchiveError(Exception): class ArchiveError(Exception):
@ -203,7 +203,7 @@ class Link:
'extension': self.extension, 'extension': self.extension,
'is_static': self.is_static, 'is_static': self.is_static,
'tags_str': self.tags, # only used to render static index in index/html.py, remove if no longer needed there 'tags_str': (self.tags or '').strip(','), # only used to render static index in index/html.py, remove if no longer needed there
'icons': None, # only used to render static index in index/html.py, remove if no longer needed there 'icons': None, # only used to render static index in index/html.py, remove if no longer needed there
'bookmarked_date': self.bookmarked_date, 'bookmarked_date': self.bookmarked_date,
@ -325,13 +325,11 @@ class Link:
### Pretty Printing Helpers ### Pretty Printing Helpers
@property @property
def bookmarked_date(self) -> Optional[str]: def bookmarked_date(self) -> Optional[str]:
from ..util import ts_to_date max_ts = (datetime.now(timezone.utc) + timedelta(days=30)).timestamp()
max_ts = (datetime.now() + timedelta(days=30)).timestamp()
if self.timestamp and self.timestamp.replace('.', '').isdigit(): if self.timestamp and self.timestamp.replace('.', '').isdigit():
if 0 < float(self.timestamp) < max_ts: if 0 < float(self.timestamp) < max_ts:
return ts_to_date(datetime.fromtimestamp(float(self.timestamp))) return ts_to_date_str(datetime.fromtimestamp(float(self.timestamp)))
else: else:
return str(self.timestamp) return str(self.timestamp)
return None return None
@ -339,13 +337,12 @@ class Link:
@property @property
def updated_date(self) -> Optional[str]: def updated_date(self) -> Optional[str]:
from ..util import ts_to_date return ts_to_date_str(self.updated) if self.updated else None
return ts_to_date(self.updated) if self.updated else None
@property @property
def archive_dates(self) -> List[datetime]: def archive_dates(self) -> List[datetime]:
return [ return [
result.start_ts parse_date(result.start_ts)
for method in self.history.keys() for method in self.history.keys()
for result in self.history[method] for result in self.history[method]
] ]

30
archivebox/logging_util.py
View file

@ -10,7 +10,7 @@ from math import log
from multiprocessing import Process from multiprocessing import Process
from pathlib import Path from pathlib import Path
from datetime import datetime from datetime import datetime, timezone
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any, Optional, List, Dict, Union, IO, TYPE_CHECKING from typing import Any, Optional, List, Dict, Union, IO, TYPE_CHECKING
@ -138,17 +138,19 @@ class TimedProgress:
"""Show a progress bar and measure elapsed time until .end() is called""" """Show a progress bar and measure elapsed time until .end() is called"""
def __init__(self, seconds, prefix=''): def __init__(self, seconds, prefix=''):
self.SHOW_PROGRESS = SHOW_PROGRESS self.SHOW_PROGRESS = SHOW_PROGRESS
if self.SHOW_PROGRESS: if self.SHOW_PROGRESS:
self.p = Process(target=progress_bar, args=(seconds, prefix)) self.p = Process(target=progress_bar, args=(seconds, prefix))
self.p.start() self.p.start()
self.stats = {'start_ts': datetime.now(), 'end_ts': None} self.stats = {'start_ts': datetime.now(timezone.utc), 'end_ts': None}
def end(self): def end(self):
"""immediately end progress, clear the progressbar line, and save end_ts""" """immediately end progress, clear the progressbar line, and save end_ts"""
end_ts = datetime.now()
end_ts = datetime.now(timezone.utc)
self.stats['end_ts'] = end_ts self.stats['end_ts'] = end_ts
if self.SHOW_PROGRESS: if self.SHOW_PROGRESS:
@ -231,7 +233,7 @@ def progress_bar(seconds: int, prefix: str='') -> None:
def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional[str], pwd: str): def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional[str], pwd: str):
cmd = ' '.join(('archivebox', subcommand, *subcommand_args)) cmd = ' '.join(('archivebox', subcommand, *subcommand_args))
stderr('{black}[i] [{now}] ArchiveBox v{VERSION}: {cmd}{reset}'.format( stderr('{black}[i] [{now}] ArchiveBox v{VERSION}: {cmd}{reset}'.format(
now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), now=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
VERSION=VERSION, VERSION=VERSION,
cmd=cmd, cmd=cmd,
**ANSI, **ANSI,
@ -243,7 +245,7 @@ def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional
def log_importing_started(urls: Union[str, List[str]], depth: int, index_only: bool): def log_importing_started(urls: Union[str, List[str]], depth: int, index_only: bool):
_LAST_RUN_STATS.parse_start_ts = datetime.now() _LAST_RUN_STATS.parse_start_ts = datetime.now(timezone.utc)
print('{green}[+] [{}] Adding {} links to index (crawl depth={}){}...{reset}'.format( print('{green}[+] [{}] Adding {} links to index (crawl depth={}){}...{reset}'.format(
_LAST_RUN_STATS.parse_start_ts.strftime('%Y-%m-%d %H:%M:%S'), _LAST_RUN_STATS.parse_start_ts.strftime('%Y-%m-%d %H:%M:%S'),
len(urls) if isinstance(urls, list) else len(urls.split('\n')), len(urls) if isinstance(urls, list) else len(urls.split('\n')),
@ -256,7 +258,7 @@ def log_source_saved(source_file: str):
print(' > Saved verbatim input to {}/{}'.format(SOURCES_DIR_NAME, source_file.rsplit('/', 1)[-1])) print(' > Saved verbatim input to {}/{}'.format(SOURCES_DIR_NAME, source_file.rsplit('/', 1)[-1]))
def log_parsing_finished(num_parsed: int, parser_name: str): def log_parsing_finished(num_parsed: int, parser_name: str):
_LAST_RUN_STATS.parse_end_ts = datetime.now() _LAST_RUN_STATS.parse_end_ts = datetime.now(timezone.utc)
print(' > Parsed {} URLs from input ({})'.format(num_parsed, parser_name)) print(' > Parsed {} URLs from input ({})'.format(num_parsed, parser_name))
def log_deduping_finished(num_new_links: int): def log_deduping_finished(num_new_links: int):
@ -270,7 +272,7 @@ def log_crawl_started(new_links):
### Indexing Stage ### Indexing Stage
def log_indexing_process_started(num_links: int): def log_indexing_process_started(num_links: int):
start_ts = datetime.now() start_ts = datetime.now(timezone.utc)
_LAST_RUN_STATS.index_start_ts = start_ts _LAST_RUN_STATS.index_start_ts = start_ts
print() print()
print('{black}[*] [{}] Writing {} links to main index...{reset}'.format( print('{black}[*] [{}] Writing {} links to main index...{reset}'.format(
@ -281,7 +283,7 @@ def log_indexing_process_started(num_links: int):
def log_indexing_process_finished(): def log_indexing_process_finished():
end_ts = datetime.now() end_ts = datetime.now(timezone.utc)
_LAST_RUN_STATS.index_end_ts = end_ts _LAST_RUN_STATS.index_end_ts = end_ts
@ -297,7 +299,8 @@ def log_indexing_finished(out_path: str):
### Archiving Stage ### Archiving Stage
def log_archiving_started(num_links: int, resume: Optional[float]=None): def log_archiving_started(num_links: int, resume: Optional[float]=None):
start_ts = datetime.now()
start_ts = datetime.now(timezone.utc)
_LAST_RUN_STATS.archiving_start_ts = start_ts _LAST_RUN_STATS.archiving_start_ts = start_ts
print() print()
if resume: if resume:
@ -315,7 +318,8 @@ def log_archiving_started(num_links: int, resume: Optional[float]=None):
)) ))
def log_archiving_paused(num_links: int, idx: int, timestamp: str): def log_archiving_paused(num_links: int, idx: int, timestamp: str):
end_ts = datetime.now()
end_ts = datetime.now(timezone.utc)
_LAST_RUN_STATS.archiving_end_ts = end_ts _LAST_RUN_STATS.archiving_end_ts = end_ts
print() print()
print('\n{lightyellow}[X] [{now}] Downloading paused on link {timestamp} ({idx}/{total}){reset}'.format( print('\n{lightyellow}[X] [{now}] Downloading paused on link {timestamp} ({idx}/{total}){reset}'.format(
@ -330,7 +334,8 @@ def log_archiving_paused(num_links: int, idx: int, timestamp: str):
print(' archivebox update --resume={}'.format(timestamp)) print(' archivebox update --resume={}'.format(timestamp))
def log_archiving_finished(num_links: int): def log_archiving_finished(num_links: int):
end_ts = datetime.now()
end_ts = datetime.now(timezone.utc)
_LAST_RUN_STATS.archiving_end_ts = end_ts _LAST_RUN_STATS.archiving_end_ts = end_ts
assert _LAST_RUN_STATS.archiving_start_ts is not None assert _LAST_RUN_STATS.archiving_start_ts is not None
seconds = end_ts.timestamp() - _LAST_RUN_STATS.archiving_start_ts.timestamp() seconds = end_ts.timestamp() - _LAST_RUN_STATS.archiving_start_ts.timestamp()
@ -356,6 +361,7 @@ def log_archiving_finished(num_links: int):
def log_link_archiving_started(link: "Link", link_dir: str, is_new: bool): def log_link_archiving_started(link: "Link", link_dir: str, is_new: bool):
# [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford" # [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford"
# http://www.benstopford.com/2015/02/14/log-structured-merge-trees/ # http://www.benstopford.com/2015/02/14/log-structured-merge-trees/
# > output/archive/1478739709 # > output/archive/1478739709
@ -363,7 +369,7 @@ def log_link_archiving_started(link: "Link", link_dir: str, is_new: bool):
print('\n[{symbol_color}{symbol}{reset}] [{symbol_color}{now}{reset}] "{title}"'.format( print('\n[{symbol_color}{symbol}{reset}] [{symbol_color}{now}{reset}] "{title}"'.format(
symbol_color=ANSI['green' if is_new else 'black'], symbol_color=ANSI['green' if is_new else 'black'],
symbol='+' if is_new else '√', symbol='+' if is_new else '√',
now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), now=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
title=link.title or link.base_url, title=link.title or link.base_url,
**ANSI, **ANSI,
)) ))

1
archivebox/main.py
View file

@ -585,6 +585,7 @@ def add(urls: Union[str, List[str]],
update_all: bool=not ONLY_NEW, update_all: bool=not ONLY_NEW,
index_only: bool=False, index_only: bool=False,
overwrite: bool=False, overwrite: bool=False,
# duplicate: bool=False, # TODO: reuse the logic from admin.py resnapshot to allow adding multiple snapshots by appending timestamp automatically
init: bool=False, init: bool=False,
extractors: str="", extractors: str="",
parser: str="auto", parser: str="auto",

6
archivebox/parsers/__init__.py
View file

@ -11,7 +11,7 @@ import re
from io import StringIO from io import StringIO
from typing import IO, Tuple, List, Optional from typing import IO, Tuple, List, Optional
from datetime import datetime from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from ..system import atomic_write from ..system import atomic_write
@ -147,7 +147,7 @@ def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None,
@enforce_types @enforce_types
def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir: Path=OUTPUT_DIR) -> str: def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir: Path=OUTPUT_DIR) -> str:
ts = str(datetime.now().timestamp()).split('.', 1)[0] ts = str(datetime.now(timezone.utc).timestamp()).split('.', 1)[0]
source_path = str(out_dir / SOURCES_DIR_NAME / filename.format(ts=ts)) source_path = str(out_dir / SOURCES_DIR_NAME / filename.format(ts=ts))
atomic_write(source_path, raw_text) atomic_write(source_path, raw_text)
log_source_saved(source_file=source_path) log_source_saved(source_file=source_path)
@ -157,7 +157,7 @@ def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir:
@enforce_types @enforce_types
def save_file_as_source(path: str, timeout: int=TIMEOUT, filename: str='{ts}-{basename}.txt', out_dir: Path=OUTPUT_DIR) -> str: def save_file_as_source(path: str, timeout: int=TIMEOUT, filename: str='{ts}-{basename}.txt', out_dir: Path=OUTPUT_DIR) -> str:
"""download a given url's content into output/sources/domain-<timestamp>.txt""" """download a given url's content into output/sources/domain-<timestamp>.txt"""
ts = str(datetime.now().timestamp()).split('.', 1)[0] ts = str(datetime.now(timezone.utc).timestamp()).split('.', 1)[0]
source_path = str(OUTPUT_DIR / SOURCES_DIR_NAME / filename.format(basename=basename(path), ts=ts)) source_path = str(OUTPUT_DIR / SOURCES_DIR_NAME / filename.format(basename=basename(path), ts=ts))
if any(path.startswith(s) for s in ('http://', 'https://', 'ftp://')): if any(path.startswith(s) for s in ('http://', 'https://', 'ftp://')):

4
archivebox/parsers/generic_html.py
View file

@ -4,7 +4,7 @@ __package__ = 'archivebox.parsers'
import re import re
from typing import IO, Iterable, Optional from typing import IO, Iterable, Optional
from datetime import datetime from datetime import datetime, timezone
from ..index.schema import Link from ..index.schema import Link
from ..util import ( from ..util import (
@ -46,7 +46,7 @@ def parse_generic_html_export(html_file: IO[str], root_url: Optional[str]=None,
for archivable_url in re.findall(URL_REGEX, url): for archivable_url in re.findall(URL_REGEX, url):
yield Link( yield Link(
url=htmldecode(archivable_url), url=htmldecode(archivable_url),
timestamp=str(datetime.now().timestamp()), timestamp=str(datetime.now(timezone.utc).timestamp()),
title=None, title=None,
tags=None, tags=None,
sources=[html_file.name], sources=[html_file.name],

4
archivebox/parsers/generic_json.py
View file

@ -3,7 +3,7 @@ __package__ = 'archivebox.parsers'
import json import json
from typing import IO, Iterable from typing import IO, Iterable
from datetime import datetime from datetime import datetime, timezone
from ..index.schema import Link from ..index.schema import Link
from ..util import ( from ..util import (
@ -30,7 +30,7 @@ def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
raise Exception('JSON must contain URL in each entry [{"url": "http://...", ...}, ...]') raise Exception('JSON must contain URL in each entry [{"url": "http://...", ...}, ...]')
# Parse the timestamp # Parse the timestamp
ts_str = str(datetime.now().timestamp()) ts_str = str(datetime.now(timezone.utc).timestamp())
if link.get('timestamp'): if link.get('timestamp'):
# chrome/ff histories use a very precise timestamp # chrome/ff histories use a very precise timestamp
ts_str = str(link['timestamp'] / 10000000) ts_str = str(link['timestamp'] / 10000000)

8
archivebox/parsers/generic_txt.py
View file

@ -4,7 +4,7 @@ __description__ = 'Plain Text'
import re import re
from typing import IO, Iterable from typing import IO, Iterable
from datetime import datetime from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from ..index.schema import Link from ..index.schema import Link
@ -29,7 +29,7 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
if Path(line).exists(): if Path(line).exists():
yield Link( yield Link(
url=line, url=line,
timestamp=str(datetime.now().timestamp()), timestamp=str(datetime.now(timezone.utc).timestamp()),
title=None, title=None,
tags=None, tags=None,
sources=[text_file.name], sources=[text_file.name],
@ -42,7 +42,7 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
for url in re.findall(URL_REGEX, line): for url in re.findall(URL_REGEX, line):
yield Link( yield Link(
url=htmldecode(url), url=htmldecode(url),
timestamp=str(datetime.now().timestamp()), timestamp=str(datetime.now(timezone.utc).timestamp()),
title=None, title=None,
tags=None, tags=None,
sources=[text_file.name], sources=[text_file.name],
@ -54,7 +54,7 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
for sub_url in re.findall(URL_REGEX, line[1:]): for sub_url in re.findall(URL_REGEX, line[1:]):
yield Link( yield Link(
url=htmldecode(sub_url), url=htmldecode(sub_url),
timestamp=str(datetime.now().timestamp()), timestamp=str(datetime.now(timezone.utc).timestamp()),
title=None, title=None,
tags=None, tags=None,
sources=[text_file.name], sources=[text_file.name],

4
archivebox/parsers/pinboard_rss.py
View file

@ -2,7 +2,7 @@ __package__ = 'archivebox.parsers'
from typing import IO, Iterable from typing import IO, Iterable
from datetime import datetime from datetime import datetime, timezone
from xml.etree import ElementTree from xml.etree import ElementTree
@ -36,7 +36,7 @@ def parse_pinboard_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
if ts_str: if ts_str:
time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z") time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
else: else:
time = datetime.now() time = datetime.now(timezone.utc)
yield Link( yield Link(
url=htmldecode(url), url=htmldecode(url),

4
archivebox/parsers/url_list.py
View file

@ -4,7 +4,7 @@ __description__ = 'URL list'
import re import re
from typing import IO, Iterable from typing import IO, Iterable
from datetime import datetime from datetime import datetime, timezone
from ..index.schema import Link from ..index.schema import Link
from ..util import ( from ..util import (
@ -25,7 +25,7 @@ def parse_url_list(text_file: IO[str], **_kwargs) -> Iterable[Link]:
yield Link( yield Link(
url=url, url=url,
timestamp=str(datetime.now().timestamp()), timestamp=str(datetime.now(timezone.utc).timestamp()),
title=None, title=None,
tags=None, tags=None,
sources=[text_file.name], sources=[text_file.name],

292
archivebox/templates/admin/base.html
View file

@ -4,78 +4,69 @@
<!DOCTYPE html> <!DOCTYPE html>
<html lang="{{ LANGUAGE_CODE|default:"en-us" }}" {% if LANGUAGE_BIDI %}dir="rtl"{% endif %}> <html lang="{{ LANGUAGE_CODE|default:"en-us" }}" {% if LANGUAGE_BIDI %}dir="rtl"{% endif %}>
<head> <head>
<title>{% block title %}{% endblock %} | ArchiveBox</title> <title>{% block title %}Home{% endblock %} | ArchiveBox</title>
<link rel="stylesheet" type="text/css" href="{% block stylesheet %}{% static "admin/css/base.css" %}{% endblock %}">
{% block extrastyle %}{% endblock %} {% block blockbots %}
{% if LANGUAGE_BIDI %}<link rel="stylesheet" type="text/css" href="{% block stylesheet_rtl %}{% static "admin/css/rtl.css" %}{% endblock %}">{% endif %} <meta name="robots" content="NONE,NOARCHIVE">
{% block extrahead %}{% endblock %} {% endblock %}
{% block responsive %}
<link rel="stylesheet" type="text/css" href="{% block stylesheet %}{% static "admin/css/base.css" %}{% endblock %}">
{% block extrastyle %}{% endblock %}
{% if LANGUAGE_BIDI %}
<link rel="stylesheet" type="text/css" href="{% block stylesheet_rtl %}{% static "admin/css/rtl.css" %}{% endblock %}">
{% endif %}
{% block responsive %}
<meta name="viewport" content="user-scalable=no, width=device-width, initial-scale=1.0, maximum-scale=1.0"> <meta name="viewport" content="user-scalable=no, width=device-width, initial-scale=1.0, maximum-scale=1.0">
<link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive.css" %}"> <link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive.css" %}">
{% if LANGUAGE_BIDI %}<link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive_rtl.css" %}">{% endif %} {% if LANGUAGE_BIDI %}
{% endblock %} <link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive_rtl.css" %}">
{% block blockbots %}<meta name="robots" content="NONE,NOARCHIVE">{% endblock %} {% endif %}
<link rel="stylesheet" type="text/css" href="{% static "admin.css" %}"> {% endblock %}
</head>
{% load i18n %}
<body class="{% if is_popup %}popup {% endif %}{% block bodyclass %}{% endblock %}" <link rel="stylesheet" type="text/css" href="{% static "admin.css" %}">
data-admin-utc-offset="{% now "Z" %}">
<style>
/* Loading Progress Bar */
#progress {
position: absolute;
z-index: 1000;
top: 0px;
left: -6px;
width: 2%;
opacity: 1;
height: 2px;
background: #1a1a1a;
border-radius: 1px;
transition: width 4s ease-out, opacity 400ms linear;
}
@-moz-keyframes bugfix { from { padding-right: 1px ; } to { padding-right: 0; } }
</style>
<script> <script>
// Page Loading Bar function selectSnapshotListView(e) {
window.loadStart = function(distance) { e && e.stopPropagation()
var distance = distance || 0; e && e.preventDefault()
// only add progrstess bar if not already present console.log('Switching to Snapshot list view...')
if (django.jQuery("#loading-bar").length == 0) { localStorage.setItem('preferred_snapshot_view_mode', 'list')
django.jQuery("body").add("<div id=\"loading-bar\"></div>"); window.location = "{% url 'admin:core_snapshot_changelist' %}" + document.location.search
return false
} }
if (django.jQuery("#progress").length === 0) { function selectSnapshotGridView(e) {
django.jQuery("body").append(django.jQuery("<div></div>").attr("id", "progress")); e && e.stopPropagation()
let last_distance = (distance || (30 + (Math.random() * 30))) e && e.preventDefault()
django.jQuery("#progress").width(last_distance + "%"); console.log('Switching to Snapshot grid view...')
setInterval(function() { localStorage.setItem('preferred_snapshot_view_mode', 'grid')
last_distance += Math.random() window.location = "{% url 'admin:grid' %}" + document.location.search
django.jQuery("#progress").width(last_distance + "%"); return false
}, 1000)
} }
};
window.loadFinish = function() { const preferred_view = localStorage.getItem('preferred_snapshot_view_mode') || 'unset'
django.jQuery("#progress").width("101%").delay(200).fadeOut(400, function() { const current_view = (
django.jQuery(this).remove(); window.location.pathname === "{% url 'admin:core_snapshot_changelist' %}"
}); ? 'list'
}; : 'grid')
window.loadStart(); console.log('Preferred snapshot view is:', preferred_view, 'Current view mode is:', current_view)
window.addEventListener('beforeunload', function() {window.loadStart(27)});
document.addEventListener('DOMContentLoaded', function() {window.loadFinish()}); if (preferred_view === 'grid' && current_view !== 'grid') {
selectSnapshotGridView()
}
</script> </script>
{% block extrahead %}{% endblock %}
</head>
<!-- Container -->
<div id="container">
<body class="{% if is_popup %}popup {% endif %}{% block bodyclass %}{% endblock %}" data-admin-utc-offset="{% now "Z" %}">
{% include 'progressbar.html' %}
<div id="container">
{% if not is_popup %} {% if not is_popup %}
<!-- Header -->
<div id="header"> <div id="header">
<div id="branding"> <div id="branding">
<h1 id="site-name"> <h1 id="site-name">
@ -83,11 +74,6 @@
<img src="{% static 'archive.png' %}" id="logo"> <img src="{% static 'archive.png' %}" id="logo">
ArchiveBox ArchiveBox
</a> </a>
&nbsp; &nbsp;
<small style="display: inline-block;margin-top: 2px;font-size: 18px;opacity: 0.8;">
<a><span id="snapshotListView" style="cursor: pointer">☰</span></a> |
<a><span id="snapshotGridView"style="letter-spacing: -.4em; cursor: pointer;">⣿⣿</span></a>
</small>
</h1> </h1>
</div> </div>
{% block usertools %} {% block usertools %}
@ -97,7 +83,7 @@
{% endblock %} {% endblock %}
{% block nav-global %}{% endblock %} {% block nav-global %}{% endblock %}
</div> </div>
<!-- END Header -->
{% block breadcrumbs %} {% block breadcrumbs %}
<div class="breadcrumbs"> <div class="breadcrumbs">
<a href="{% url 'admin:index' %}">{% trans 'Home' %}</a> <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
@ -108,14 +94,21 @@
{% block messages %} {% block messages %}
{% if messages %} {% if messages %}
<ul class="messagelist">{% for message in messages %} <ul class="messagelist">
{% for message in messages %}
<li{% if message.tags %} class="{{ message.tags }}"{% endif %}>{{ message|capfirst }}</li> <li{% if message.tags %} class="{{ message.tags }}"{% endif %}>{{ message|capfirst }}</li>
{% endfor %}</ul> {% endfor %}
</ul>
{% endif %} {% endif %}
{% endblock messages %} {% endblock messages %}
<!-- Content -->
<div id="content" class="{% block coltype %}colM{% endblock %}"> <div id="content" class="{% block coltype %}colM{% endblock %}">
{% if opts.model_name == 'snapshot' and cl %}
<small id="snapshot-view-mode">
<a href="#list" title="List view" id="snapshot-view-list">☰</a> |
<a href="#grid" title="Grid view" id="snapshot-view-grid" style="letter-spacing: -.4em;">⣿⣿</a>
</small>
{% endif %}
{% block pretitle %}{% endblock %} {% block pretitle %}{% endblock %}
{% block content_title %}{# {% if title %}<h1>{{ title }}</h1>{% endif %} #}{% endblock %} {% block content_title %}{# {% if title %}<h1>{{ title }}</h1>{% endif %} #}{% endblock %}
{% block content %} {% block content %}
@ -125,107 +118,114 @@
{% block sidebar %}{% endblock %} {% block sidebar %}{% endblock %}
<br class="clear"> <br class="clear">
</div> </div>
<!-- END Content -->
{% block footer %}<div id="footer"></div>{% endblock %} {% block footer %}<div id="footer"></div>{% endblock %}
</div> </div>
<!-- END Container -->
<script> <script>
(function ($) { $ = django.jQuery;
$.fn.reverse = [].reverse; $.fn.reverse = [].reverse;
// hide images that fail to load
document.querySelector('body').addEventListener('error', function (e) {
e.target.style.opacity = 0;
}, true)
// setup timezone
{% get_current_timezone as TIME_ZONE %}
window.TIME_ZONE = '{{TIME_ZONE}}'
window.setCookie = function(name, value, days) {
let expires = ""
if (days) {
const date = new Date()
date.setTime(date.getTime() + (days*24*60*60*1000))
expires = "; expires=" + date.toUTCString()
}
document.cookie = name + "=" + (value || "") + expires + "; path=/"
}
function setTimeOffset() {
if (window.GMT_OFFSET) return
window.GMT_OFFSET = -(new Date).getTimezoneOffset()
window.setCookie('GMT_OFFSET', window.GMT_OFFSET, 365)
}
// change the admin actions button from a dropdown to buttons across
function fix_actions() { function fix_actions() {
var container = $('div.actions'); const container = $('div.actions')
if (container.find('select[name=action] option').length < 10) { // too many actions to turn into buttons
container.find('label:nth-child(1), button[value=0]').hide(); if (container.find('select[name=action] option').length >= 11) return
var buttons = $('<div></div>') // hide the empty default option thats just a placeholder with no value
.appendTo(container) container.find('label:nth-child(1), button[value=0]').hide()
const buttons = $('<div></div>')
.insertAfter('div.actions button[type=submit]')
.css('display', 'inline') .css('display', 'inline')
.addClass('class', 'action-buttons'); .addClass('class', 'action-buttons');
container.find('select[name=action] option:gt(0)').reverse().each(function () { // for each action in the dropdown, turn it into a button instead
const name = this.value container.find('select[name=action] option:gt(0)').each(function () {
const action_type = this.value
$('<button>') $('<button>')
.appendTo(buttons)
.attr('name', this.value)
.attr('type', 'button') .attr('type', 'button')
.attr('name', action_type)
.addClass('button') .addClass('button')
.text(this.text) .text(this.text)
.click(function (e) { .click(function (e) {
e.preventDefault() e.preventDefault()
e.stopPropagation() e.stopPropagation()
container.find('select')
.find(':selected').removeAttr('selected').end() const num_selected = document.querySelector('.action-counter').innerText.split(' ')[0]
.find('[value=' + name + ']').attr('selected', 'selected').click();
$('#changelist-form button[name="index"]').click(); if (action_type === 'overwrite_snapshots') {
const message = (
'Are you sure you want to re-archive (overwrite) ' + num_selected + ' Snapshots?\n\n' +
'This will delete all previously saved files from these Snapshots and re-archive them from scratch.\n\n'
)
if (!window.confirm(message)) return false
}
if (action_type === 'delete_snapshots') {
const message = (
'Are you sure you want to permanently delete ' + num_selected + ' Snapshots?\n\n' +
'They will be removed from your index, and all their Snapshot content on disk will be permanently deleted.'
)
if (!window.confirm(message)) return false
}
// select the action button from the dropdown
container.find('select[name=action]')
.find('op:selected').removeAttr('selected').end()
.find('[value=' + action_type + ']').attr('selected', 'selected').click()
// click submit & replace the archivebox logo with a spinner
$('#changelist-form button[name="index"]').click()
document.querySelector('#logo').outerHTML = '<div class="loader"></div>' document.querySelector('#logo').outerHTML = '<div class="loader"></div>'
return false return false
}); })
}); .appendTo(buttons)
} })
}; console.log('Converted', buttons.children().length, 'admin actions from dropdown to buttons')
function redirectWithQuery(uri){
uri_query = uri + document.location.search;
window.location = uri_query;
};
function selectSnapshotListView(){
localStorage.setItem('currentSnapshotView', 'List');
redirectWithQuery("{% url 'admin:core_snapshot_changelist' %}");
};
function selectSnapshotGridView(){
localStorage.setItem('currentSnapshotView', 'Grid');
redirectWithQuery("{% url 'admin:grid' %}");
};
function setPreferredSnapshotView(view){
urlPath = window.location.pathname;
if((view==="Grid") && urlPath == "{% url 'admin:core_snapshot_changelist' %}"){
selectSnapshotGridView();
} }
{% comment %} function setupSnapshotGridListToggle() {
else if((view==="List") && urlPath == "{% url 'admin:grid' %}"){ $("#snapshot-view-list").click(selectSnapshotListView)
selectSnapshotListView(); $("#snapshot-view-grid").click(selectSnapshotGridView)
} $('#changelist-form .card input:checkbox').change(function() {
{% endcomment %} if ($(this).is(':checked'))
}; $(this).parents('.card').addClass('selected-card')
function setupSnapshotViews() {
const preferredSnapshotView = localStorage.getItem('currentSnapshotView');
setPreferredSnapshotView(preferredSnapshotView);
$( document ).ready(function() {
$("#snapshotListView").click(function() {
selectSnapshotListView();
});
$("#snapshotGridView").click(function() {
selectSnapshotGridView();
});
$('input:checkbox').change(function(){
if($(this).is(':checked'))
$(this).parent().parent().parent().parent().addClass('selected-card');
else else
$(this).parent().parent().parent().parent().removeClass('selected-card') $(this).parents('.card').removeClass('selected-card')
}); })
});
}; };
$(function () { $(document).ready(function() {
fix_actions(); fix_actions()
setupSnapshotViews(); setupSnapshotGridListToggle()
}); setTimeOffset()
})(django.jQuery); })
</script> </script>
</body> </body>
</html> </html>

289
archivebox/templates/admin/snapshots_grid.html
View file

@ -3,125 +3,140 @@
{% block extrastyle %} {% block extrastyle %}
<style> <style>
* { #changelist-search #searchbar {
-webkit-box-sizing: border-box; min-height: 24px;
-moz-box-sizing: border-box; }
box-sizing: border-box; .cards {
} padding-top: 10px;
a {
text-decoration: none;
color: orange;
}
h2 {
color: #000;
margin: 2rem 0 .5rem;
font-size: 1.25rem;
font-weight: 400;
{% comment %} text-transform: uppercase; {% endcomment %}
}
card.img {
display: block;
border: 0;
width: 100%;
height: auto;
}
/*************************** Cards *******************************/
.cards {
display: grid; display: grid;
grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); /* see notes below */ grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); /* see notes below */
grid-auto-rows: minmax(200px, auto); grid-auto-rows: minmax(200px, auto);
grid-gap: 1rem; grid-gap: 14px 14px;
} }
.card { .cards .card {
/*height: 200px;*/
/*background: red;*/
border: 2px solid #e7e7e7;
border-radius: 4px;
-webkit-box-shadow: 0 2px 2px rgba(0, 0, 0, 0.15);
box-shadow: 0 2px 2px rgba(0, 0, 0, 0.15);
display: flex;
/* -webkit-box-orient: vertical; */
/* -webkit-box-direction: normal; */
-ms-flex-direction: column;
flex-direction: column;
position: relative; position: relative;
max-height: 380px;
overflow: hidden;
background-color: #fffcfc;
border: 1px solid #f1efef;
border-radius: 4px;
box-shadow: 4px 4px 2px 2px rgba(0, 0, 0, 0.01);
text-align: center;
color: #5d5e5e; color: #5d5e5e;
} /* li item */ }
.thumbnail img { .cards .card.selected-card {
height: 100%; border: 3px solid #2196f3;
box-sizing: border-box; box-shadow: 2px 3px 6px 2px rgba(0, 0, 221, 0.14);
max-width: 100%; }
max-height: 100%;
.cards .card .card-thumbnail {
display: block;
width: 100%; width: 100%;
} height: 345px;
overflow: hidden;
border-radius: 4px;
background-color: #fffcfc;
}
.card-content { .cards .card .card-thumbnail img {
font-size: .75rem; width: 100%;
padding: .5rem; height: auto;
display: flex; border: 0;
-webkit-box-orient: vertical; }
-webkit-box-direction: normal; .cards .card .card-thumbnail.missing img {
-ms-flex-direction: column; opacity: 0.03;
flex-direction: column; width: 20%;
-webkit-box-flex: 1; height: auto;
-ms-flex: 1; margin-top: 84px;
flex: 1; }
} .cards .card .card-tags {
width: 100%;
}
.cards .card .card-tags span {
display: inline-block;
padding: 2px 5px;
border-radius: 5px;
opacity: 0.95;
background-color: #bfdfff;
color: #679ac2;
font-size: 12px;
margin-bottom: 3px;
}
.card-content h4{ .cards .card .card-footer {
vertical-align:bottom; width: 100%;
margin: 1.2em 0 0em 0;
}
.category {
font-size: .75rem;
text-transform: uppercase;
}
.category {
position: absolute; position: absolute;
top: 5%; bottom: 0;
right: 0; text-align: center;
color: #fff; }
background: #e74c3c; .cards .card .card-title {
padding: 10px 15px; padding: 4px 5px;
font-size: 10px; background-color: #fffcfc;
/*height: 50px;
vertical-align: middle;
line-height: 50px;*/
}
.cards .card .card-title h4 {
color: initial;
display: block;
vertical-align: middle;
line-height: normal;
margin: 0px;
padding: 5px 0px;
font-size: 13.5px;
font-weight: 400;
word-break: break-all;
overflow: hidden;
text-overflow: ellipsis;
max-height: 46px;
}
.cards .card .card-title h4 .title-text {
user-select: all;
}
.cards .card .card-title .link-favicon {
height: 15px;
margin: 2px;
vertical-align: -5px;
display: inline-block;
}
.cards .card .card-info {
padding: 2px 4px;
/*border-top: 1px solid #ddd;*/
background-color: #fffcfc;
font-size: 11px;
color: #333;
}
.cards .card .card-info input[type=checkbox] {
float: right;
width: 18px;
height: 18px;
}
.cards .card .card-info label {
display: inline-block;
height: 20px;
width: 145px;
font-size: 11px;
}
.cards .card .card-info .timestamp {
font-weight: 600; font-weight: 600;
text-transform: uppercase; }
} .cards .card .card-footer code {
display: inline-block;
.category__01 { width: 100%;
background-color: #50c6db; margin-top: 2px;
font-size: 10px;
} opacity: 0.4;
user-select: all;
.tags{ white-space: nowrap;
opacity: 0.8; overflow: hidden;
} text-overflow: ellipsis;
}
footer {
border-top: 2px solid #e7e7e7;
{% comment %} margin: .5rem 0 0; {% endcomment %}
{% comment %} min-height: 30px; {% endcomment %}
font-size: .5rem;
}
.post-meta {
padding: .3rem;
}
.comments {
margin-left: .5rem;
}
.selected-card{
border: 5px solid #ffaa31;
}
</style> </style>
@ -130,33 +145,47 @@ footer {
{% block content %} {% block content %}
<section class="cards"> <section class="cards">
{% for obj in results %} {% for obj in results %}
<article class="card"> <div class="card">
<picture class="thumbnail"> <div class="card-info">
<a href="/{{obj.archive_path}}/index.html">
<img class="category__01" src="{% snapshot_image obj%}" alt="" />
</a>
</picture>
<div class="card-content">
{% if obj.tags_str %}
<p class="category category__01 tags">{{obj.tags_str}}</p>
{% endif %}
{% if obj.title %}
<a href="{% url 'admin:core_snapshot_change' obj.id %}"> <a href="{% url 'admin:core_snapshot_change' obj.id %}">
<h4>{{obj.title|truncatechars:55 }}</h4> <span class="timestamp">{{obj.added}}</span>
</a> </a>
{% endif %} &nbsp; &nbsp;
{% comment %} <p> TEXT If needed.</p> {% endcomment %} <label>
</div><!-- .card-content --> <span class="num_outputs">📄 &nbsp; {{obj.num_outputs}}</span> &nbsp; &nbsp;
<footer> <span>🗄&nbsp; {{ obj.archive_size | file_size }}</span>
<div class="post-meta"> <input type="checkbox" name="_selected_action" value="{{obj.pk}}"/>
<span style="float:right;"><input type="checkbox" name="_selected_action" value="{{obj.pk}}" class="action-select"></span> </label>
<span class="timestamp">&#128337 {{obj.added}}</span> </div>
<span class="comments">📖{{obj.num_outputs}}</span> <a href="/{{obj.archive_path}}/index.html" class="card-thumbnail {% if not obj.thumbnail_url %}missing{% endif %}">
<span>🗄️{{ obj.archive_size | file_size }}</span> <img src="{{obj.thumbnail_url|default:'/static/spinner.gif' }}" alt="{{obj.title|default:'Not yet archived...'}}" />
</a>
<div class="card-footer">
{% if obj.tags_str %}
<div class="card-tags">
{% for tag in obj.tags_str|split:',' %}
<span>
{{tag}}
</span>
{% endfor %}
</div>
{% endif %}
<div class="card-title" title="{{obj.title}}">
<a href="/{{obj.archive_path}}/index.html">
<h4>
{% if obj.is_archived %}
<img src="/{{obj.archive_path}}/favicon.ico" onerror="this.style.display='none'" class="link-favicon" decoding="async"/>
{% else %}
<img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async"/>
{% endif %}
<span class="title-text">{{obj.title|default:'Pending...' }}</span>
</h4>
</a>
<code title="{{obj.url}}">{{obj.url}}</code>
</div>
</div>
</div> </div>
</footer>
</article>
{% endfor %} {% endfor %}
</section> </section>
<br/>
{% endblock %} {% endblock %}

2
archivebox/templates/core/add.html
View file

@ -49,7 +49,7 @@
<script> <script>
document.getElementById('add-form').addEventListener('submit', function(event) { document.getElementById('add-form').addEventListener('submit', function(event) {
setTimeout(function() { setTimeout(function() {
document.getElementById('add-form').innerHTML = '<center><h3>Adding URLs to index and running archive methods...<h3><br/><div class="loader"></div><br/>Check server log or <a href="/admin/core/archiveresult/?o=-1">Outputs page</a> for progress...</center>' document.getElementById('add-form').innerHTML = '<center><h3>Adding URLs to index and running archive methods...<h3><br/><div class="loader"></div><br/>Check the server log or the <a href="/admin/core/archiveresult/?o=-1">Log</a> page for progress...</center>'
document.getElementById('delay-warning').style.display = 'block' document.getElementById('delay-warning').style.display = 'block'
}, 200) }, 200)
return true return true

27
archivebox/templates/core/base.html
View file

@ -2,35 +2,17 @@
<!DOCTYPE html> <!DOCTYPE html>
<html lang="en"> <html lang="en">
<head>
<head>
<title>Archived Sites</title> <title>Archived Sites</title>
<meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1"> <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="{% static 'admin/css/base.css' %}"> <link rel="stylesheet" href="{% static 'admin/css/base.css' %}">
<link rel="stylesheet" type="text/css" href="{% static 'admin.css' %}"> <link rel="stylesheet" href="{% static 'admin.css' %}">
<link rel="stylesheet" href="{% static 'bootstrap.min.css' %}"> <link rel="stylesheet" href="{% static 'bootstrap.min.css' %}">
<link rel="stylesheet" href="{% static 'jquery.dataTables.min.css' %}" />
<script src="{% static 'jquery.min.js' %}"></script>
{% block extra_head %} {% block extra_head %}
{% endblock %} {% endblock %}
<script src="{% static 'jquery.min.js' %}"></script>
<script src="{% static 'jquery.dataTables.min.js' %}"></script>
<script>
document.addEventListener('error', function (e) {
e.target.style.opacity = 0;
}, true)
jQuery(document).ready(function () {
jQuery('#table-bookmarks').DataTable({
searching: false,
paging: false,
stateSave: true, // save state (filtered input, number of entries shown, etc) in localStorage
dom: '<lf<t>ip>', // how to show the table and its helpers (filter, etc) in the DOM
order: [[0, 'desc']],
iDisplayLength: 100,
});
});
</script>
<base href="{% url 'Home' %}">
</head> </head>
<body> <body>
<div id="container"> <div id="container">
@ -48,6 +30,7 @@
</div> </div>
<div id="content" class="flex"> <div id="content" class="flex">
{% block body %} {% block body %}
{% endblock %} {% endblock %}
</div> </div>
{% block footer %} {% block footer %}

41
archivebox/templates/core/index_row.html
View file

@ -1,37 +1,44 @@
{% load static %} {% load static tz core_tags %}
<tr> <tr>
<td title="{{link.timestamp}}"> {% if link.bookmarked_date %} {{ link.bookmarked_date }} {% else %} {{ link.added }} {% endif %} </td> <td title="Bookmarked: {{link.bookmarked_date|localtime}} ({{link.timestamp}})" data-sort="{{link.added.timestamp}}">
<td class="title-col" style="opacity: {% if link.title %}1{% else %}0.3{% endif %}"> {{ link.added|localtime }}
</td>
<td class="title-col" style="opacity: {% if link.title %}1{% else %}0.3{% endif %}" title="{{link.title|default:'Not yet archived...'}}">
{% if link.is_archived %} {% if link.is_archived %}
<a href="archive/{{link.timestamp}}/index.html"><img src="archive/{{link.timestamp}}/favicon.ico" onerror="this.style.display='none'" class="link-favicon" decoding="async"></a> <a href="/archive/{{link.timestamp}}/index.html"><img src="archive/{{link.timestamp}}/favicon.ico" onerror="this.style.display='none'" class="link-favicon" decoding="async"></a>
{% else %} {% else %}
<a href="archive/{{link.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async" style="height: 15px"></a> <a href="/archive/{{link.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async" style="height: 15px"></a>
{% endif %} {% endif %}
<a href="archive/{{link.timestamp}}/index.html" title="{{link.title|default:'Not yet archived...'}}"> <a href="/archive/{{link.timestamp}}/index.html" title="{{link.title|default:'Not yet archived...'}}">
<span data-title-for="{{link.url}}" data-archived="{{link.is_archived}}">{{link.title|default:'Loading...'|truncatechars:128}}</span> <span data-title-for="{{link.url}}" data-archived="{{link.is_archived}}">
{% if link.tags_str %} {{link.title|default:'Loading...'|truncatechars:128}}
<span class="tags" style="float: right; border-radius: 5px; background-color: #bfdfff; padding: 2px 5px; margin-left: 4px; margin-top: 1px;">
{% if link.tags_str != None %}
{{link.tags_str|default:''}}
{% else %}
{{ link.tags|default:'' }}
{% endif %}
</span> </span>
{% if link.tags_str %}
{% for tag in link.tags_str|split:',' %}
<span class="tag" style="float: right; border-radius: 5px; background-color: #bfdfff; padding: 2px 5px; margin-left: 4px; margin-top: 1px;">
{{tag}}
</span>
{% endfor %}
{% endif %} {% endif %}
</a> </a>
</td> </td>
<td> <td>
<span data-number-for="{{link.url}}" title="Fetching any missing files..."> <span data-number-for="{{link.url}}" title="Fetching any missing files...">
{% if link.icons %} {% if link.icons %}
{{link.icons}} <small style="float:right; opacity: 0.5">{{link.num_outputs}}</small> {{link.icons}}&nbsp; <small style="float:right; opacity: 0.5">{{link.num_outputs}}</small>
{% else %} {% else %}
<a href="archive/{{link.timestamp}}/index.html">📄 <a href="/archive/{{link.timestamp}}/index.html">
📄 &nbsp;
{{link.num_outputs}} <img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="files-spinner" decoding="async" style="height: 15px"/> {{link.num_outputs}} <img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="files-spinner" decoding="async" style="height: 15px"/>
</a> </a>
{% endif %} {% endif %}
</span> </span>
</td> </td>
<td style="text-align:left; word-wrap: anywhere;"><a href="{{link.url}}">{{link.url|truncatechars:128}}</a></td> <td style="text-align:left; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; title="{{link.url}}">
<a href="{{link.url}}">
{{link.url}}
</a>
</td>
</tr> </tr>

45
archivebox/templates/core/progressbar.html Normal file
View file

@ -0,0 +1,45 @@
<style>
/* Loading Progress Bar */
#progress {
position: absolute;
z-index: 1000;
top: 0px;
left: -6px;
width: 2%;
opacity: 1;
height: 2px;
background: #1a1a1a;
border-radius: 1px;
transition: width 4s ease-out, opacity 400ms linear;
}
@-moz-keyframes bugfix { from { padding-right: 1px ; } to { padding-right: 0; } }
</style>
<script>
// Page Loading Bar
window.loadStart = function(distance) {
var distance = distance || 0;
// only add progrstess bar if not already present
if (django.jQuery("#loading-bar").length == 0) {
django.jQuery("body").add("<div id=\"loading-bar\"></div>");
}
if (django.jQuery("#progress").length === 0) {
django.jQuery("body").append(django.jQuery("<div></div>").attr("id", "progress"));
let last_distance = (distance || (30 + (Math.random() * 30)))
django.jQuery("#progress").width(last_distance + "%");
setInterval(function() {
last_distance += Math.random()
django.jQuery("#progress").width(last_distance + "%");
}, 1000)
}
};
window.loadFinish = function() {
django.jQuery("#progress").width("101%").delay(200).fadeOut(400, function() {
django.jQuery(this).remove();
});
};
window.loadStart();
window.addEventListener('beforeunload', function() {window.loadStart(27)});
document.addEventListener('DOMContentLoaded', function() {window.loadFinish()});
</script>

27
archivebox/templates/core/public_index.html
View file

@ -1,12 +1,7 @@
{% extends "base.html" %} {% extends "base.html" %}
{% load static %} {% load static tz %}
{% block body %} {% block body %}
<style>
#table-bookmarks_info {
display: none;
}
</style>
<div id="toolbar"> <div id="toolbar">
<form id="changelist-search" action="{% url 'public-index' %}" method="get"> <form id="changelist-search" action="{% url 'public-index' %}" method="get">
<div> <div>
@ -19,16 +14,22 @@
onclick="location.href='{% url 'public-index' %}'" onclick="location.href='{% url 'public-index' %}'"
style="background-color: rgba(121, 174, 200, 0.8); height: 30px; font-size: 0.8em; margin-top: 12px; padding-top: 6px; float:right"> style="background-color: rgba(121, 174, 200, 0.8); height: 30px; font-size: 0.8em; margin-top: 12px; padding-top: 6px; float:right">
</input> </input>
&nbsp;
&nbsp;
{{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
&nbsp;
(Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }})
</div> </div>
</form> </form>
</div> </div>
<table id="table-bookmarks"> <div style="width: 100%; overflow-x: auto;">
<table id="table-bookmarks" style="width: 100%; table-layout: fixed">
<thead> <thead>
<tr> <tr>
<th style="width: 100px;">Bookmarked</th> <th style="width: 130px">Bookmarked</th>
<th style="width: 26vw;">Snapshot ({{page_obj.paginator.count}})</th> <th>Snapshot ({{page_obj.paginator.count}})</th>
<th style="width: 140px">Files</th> <th style="width: 258px">Files</th>
<th style="width: 16vw;whitespace:nowrap;overflow-x:hidden;">Original URL</th> <th>Original URL</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
@ -37,8 +38,9 @@
{% endfor %} {% endfor %}
</tbody> </tbody>
</table> </table>
<center> </div>
<br/> <br/>
<center>
Showing {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total Showing {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
<br/> <br/>
<span class="step-links"> <span class="step-links">
@ -58,7 +60,6 @@
<a href="{% url 'public-index' %}?page={{ page_obj.paginator.num_pages }}">last &raquo;</a> <a href="{% url 'public-index' %}?page={{ page_obj.paginator.num_pages }}">last &raquo;</a>
{% endif %} {% endif %}
</span> </span>
</span>
<br> <br>
</center> </center>
{% endblock %} {% endblock %}

118
archivebox/templates/core/snapshot.html
View file

@ -1,3 +1,5 @@
{% load tz core_tags %}
<!DOCTYPE html> <!DOCTYPE html>
<html lang="en"> <html lang="en">
<head> <head>
@ -20,7 +22,6 @@
} }
header { header {
background-color: #aa1e55; background-color: #aa1e55;
padding-bottom: 12px;
} }
small { small {
font-weight: 200; font-weight: 200;
@ -34,15 +35,15 @@
min-height: 40px; min-height: 40px;
margin: 0px; margin: 0px;
text-align: center; text-align: center;
color: white; color: #f6f6f6;
font-size: calc(11px + 0.84vw); font-size: calc(10px + 0.84vw);
font-weight: 200; font-weight: 200;
padding: 4px 4px; padding: 3px 4px;
background-color: #aa1e55; background-color: #aa1e55;
} }
.nav > div { .nav > div {
min-height: 30px; min-height: 30px;
line-height: 1.3; line-height: 1.2;
} }
.header-top a { .header-top a {
text-decoration: none; text-decoration: none;
@ -68,9 +69,14 @@
.header-archivebox img:hover { .header-archivebox img:hover {
opacity: 0.5; opacity: 0.5;
} }
.header-url small { header small code {
white-space: nowrap; white-space: nowrap;
font-weight: 200; font-weight: 200;
display: block;
margin-top: -1px;
font-size: 13px;
opacity: 0.8;
user-select: all;
} }
.header-url img { .header-url img {
height: 20px; height: 20px;
@ -90,28 +96,38 @@
.info-row .alert { .info-row .alert {
margin-bottom: 0px; margin-bottom: 0px;
} }
.row.header-bottom {
margin-left: -10px;
margin-right: -10px;
}
.header-bottom .col-lg-2 {
padding-left: 4px;
padding-right: 4px;
}
.header-bottom-frames .card { .header-bottom-frames .card {
overflow: hidden;
box-shadow: 2px 3px 14px 0px rgba(0,0,0,0.02); box-shadow: 2px 3px 14px 0px rgba(0,0,0,0.02);
margin-top: 10px; margin-bottom: 5px;
border: 1px solid rgba(0,0,0,3); border: 1px solid rgba(0,0,0,3);
border-radius: 14px; border-radius: 10px;
background-color: black; background-color: black;
overflow: hidden;
} }
.card h4 { .card h4 {
font-size: 1.4vw; font-size: 1.4vw;
} }
.card-body { .card-body {
font-size: 15px; font-size: 14px;
padding: 13px 10px; padding: 13px 10px;
padding-bottom: 6px; padding-bottom: 1px;
/* padding-left: 3px; */ /* padding-left: 3px; */
/* padding-right: 3px; */ /* padding-right: 3px; */
/* padding-bottom: 3px; */ /* padding-bottom: 3px; */
line-height: 1.1; line-height: 1;
word-wrap: break-word; word-wrap: break-word;
max-height: 102px; max-height: 102px;
overflow: hidden; overflow: hidden;
text-overflow: ellipsis;
background-color: #1a1a1a; background-color: #1a1a1a;
color: #d3d3d3; color: #d3d3d3;
} }
@ -146,22 +162,12 @@
border-top: 3px solid #aa1e55; border-top: 3px solid #aa1e55;
} }
.card.selected-card { .card.selected-card {
border: 1px solid orange; border: 2px solid orange;
box-shadow: 0px -6px 13px 1px rgba(0,0,0,0.05); box-shadow: 0px -6px 13px 1px rgba(0,0,0,0.05);
} }
.iframe-large { .iframe-large {
height: calc(100% - 40px); height: calc(100% - 40px);
} }
.pdf-frame {
transform: none;
width: 100%;
height: 160px;
margin-top: -60px;
margin-bottom: 0px;
transform: scale(1.1);
width: 100%;
margin-left: -10%;
}
img.external { img.external {
height: 30px; height: 30px;
margin-right: -10px; margin-right: -10px;
@ -185,7 +191,7 @@
} }
.header-bottom { .header-bottom {
border-top: 1px solid rgba(170, 30, 85, 0.9); border-top: 1px solid rgba(170, 30, 85, 0.9);
padding-bottom: 12px; padding-bottom: 1px;
border-bottom: 5px solid rgb(170, 30, 85); border-bottom: 5px solid rgb(170, 30, 85);
margin-bottom: -1px; margin-bottom: -1px;
@ -215,10 +221,11 @@
} }
.info-chunk { .info-chunk {
width: auto; width: auto;
display:inline-block; display: inline-block;
text-align: center; text-align: center;
margin: 10px 10px; margin: 8px 4px;
vertical-align: top; vertical-align: top;
font-size: 14px;
} }
.info-chunk .badge { .info-chunk .badge {
margin-top: 5px; margin-top: 5px;
@ -226,13 +233,12 @@
.header-bottom-frames .card-title { .header-bottom-frames .card-title {
width: 100%; width: 100%;
text-align: center; text-align: center;
font-size: 18px; font-size: 17px;
margin-bottom: 5px; margin-bottom: 0px;
display: inline-block; display: inline-block;
color: #d3d3d3; color: #d3d3d3;
font-weight: 200; font-weight: 200;
vertical-align: 0px; vertical-align: 3px;
margin-top: -6px;
} }
.header-bottom-frames .card-text { .header-bottom-frames .card-text {
width: 100%; width: 100%;
@ -277,8 +283,7 @@
<header> <header>
<div class="header-top container-fluid"> <div class="header-top container-fluid">
<div class="row nav"> <div class="row nav">
<div class="col-lg-2" style="line-height: 64px;"> <div class="col-lg-2" style="line-height: 50px; vertical-align: middle">
<a href="../../index.html" class="header-archivebox" title="Go to Main Index..."> <a href="../../index.html" class="header-archivebox" title="Go to Main Index...">
<img src="../../static/archive.png" alt="Archive Icon"> <img src="../../static/archive.png" alt="Archive Icon">
ArchiveBox ArchiveBox
@ -290,10 +295,9 @@
{{title|safe}} {{title|safe}}
&nbsp;&nbsp; &nbsp;&nbsp;
<a href="#" class="header-toggle">▾</a> <a href="#" class="header-toggle">▾</a>
<br/>
<small> <small>
<a href="{{url}}" class="header-url" title="{{url}}"> <a href="{{url}}" class="header-url" title="{{url}}">
{{url_str}} <code>{{url}}</code>
</a> </a>
</small> </small>
</div> </div>
@ -302,27 +306,25 @@
<div class="header-bottom container-fluid"> <div class="header-bottom container-fluid">
<div class="row header-bottom-info"> <div class="row header-bottom-info">
<div class="col-lg-4"> <div class="col-lg-4">
<div title="Date bookmarked or imported" class="info-chunk"> <div title="Date bookmarked or imported" class="info-chunk" title="UTC Timezone {{timestamp}}">
<h5>Added</h5> <h5>Added</h5>
{{bookmarked_date}} {{bookmarked_date}}
</div> </div>
<div title="Date first archived" class="info-chunk"> <div title="Date first archived" class="info-chunk" title="UTC Timezone">
<h5>First Archived</h5> <h5>First Archived</h5>
{{oldest_archive_date}} {{oldest_archive_date}}
</div> </div>
<div title="Date last checked" class="info-chunk"> <div title="Date last checked" class="info-chunk" title="UTC Timezone">
<h5>Last Checked</h5> <h5>Last Checked</h5>
{{updated_date}} {{updated_date}}
</div> </div>
</div> </div>
<div class="col-lg-4"> <div class="col-lg-4">
<div class="info-chunk"> <div class="info-chunk" style="max-width: 280px">
<h5>Type</h5> <h5>Tags <small title="Auto-guessed content type">({{extension}})</small></h5>
<div class="badge badge-default">{{extension}}</div> {% for tag in tags_str|split:',' %}
</div> <div class="badge badge-info" style="word-break: break-all;">{{tag}}</div>
<div class="info-chunk"> {% endfor %}
<h5>Tags</h5>
<div class="badge badge-warning">{{tags}}</div>
</div> </div>
<div class="info-chunk"> <div class="info-chunk">
<h5>Status</h5> <h5>Status</h5>
@ -330,11 +332,11 @@
</div> </div>
<div class="info-chunk"> <div class="info-chunk">
<h5>Saved</h5> <h5>Saved</h5>
✅ {{num_outputs}} ✅&nbsp; {{num_outputs}}
</div> </div>
<div class="info-chunk"> <div class="info-chunk">
<h5>Errors</h5> <h5>Errors</h5>
❌ {{num_failures}} ❌&nbsp; {{num_failures}}
</div> </div>
<div class="info-chunk"> <div class="info-chunk">
<h5>Size</h5> <h5>Size</h5>
@ -343,7 +345,7 @@
</div> </div>
<div class="col-lg-4"> <div class="col-lg-4">
<div class="info-chunk"> <div class="info-chunk">
<h5>🗃 Snapshot ID: <a href="/admin/core/snapshot/{{snapshot_id}}/change/"><code style="color: rgba(255,255,255,0.6); font-weight: 200; font-size: 12px; background-color: #1a1a1a"><b>[{{timestamp}}]</b> <small>{{snapshot_id|truncatechars:24}}</small></code></a></h5> <h5>🗃&nbsp; Snapshot: <a href="/admin/core/snapshot/{{snapshot_id}}/change/"><code style="color: rgba(255,255,255,0.6); font-weight: 200; font-size: 12px; background-color: #1a1a1a"><b>[{{timestamp}}]</b> <small>{{snapshot_id|truncatechars:24}}</small></code></a></h5>
<a href="index.json" title="JSON summary of archived link.">JSON</a> | <a href="index.json" title="JSON summary of archived link.">JSON</a> |
<a href="warc/" title="Any WARC archives for the page">WARC</a> | <a href="warc/" title="Any WARC archives for the page">WARC</a> |
<a href="media/" title="Audio, Video, and Subtitle files.">Media</a> | <a href="media/" title="Audio, Video, and Subtitle files.">Media</a> |
@ -357,7 +359,7 @@
<div class="row header-bottom-frames"> <div class="row header-bottom-frames">
<div class="col-lg-2"> <div class="col-lg-2">
<div class="card selected-card"> <div class="card selected-card">
<iframe class="card-img-top" src="{{singlefile_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe> <iframe class="card-img-top" src="{{singlefile_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body"> <div class="card-body">
<a href="{{singlefile_path}}" title="Open in new tab..." target="_blank" rel="noopener"> <a href="{{singlefile_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./singlefile.html</code></p> <p class="card-text"><code>./singlefile.html</code></p>
@ -368,7 +370,7 @@
</div> </div>
<div class="col-lg-2"> <div class="col-lg-2">
<div class="card"> <div class="card">
<iframe class="card-img-top pdf-frame" src="{{pdf_path}}" scrolling="no"></iframe> <iframe class="card-img-top pdf-frame" src="{{pdf_path}}#toolbar=0" scrolling="no" loading="lazy"></iframe>
<div class="card-body"> <div class="card-body">
<a href="{{pdf_path}}" title="Open in new tab..." target="_blank" rel="noopener"> <a href="{{pdf_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./output.pdf</code></p> <p class="card-text"><code>./output.pdf</code></p>
@ -390,7 +392,7 @@
</div> </div>
<div class="col-lg-2"> <div class="col-lg-2">
<div class="card"> <div class="card">
<iframe class="card-img-top" src="{{archive_url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe> <iframe class="card-img-top" src="{{archive_url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body"> <div class="card-body">
<a href="{{archive_url}}" title="Open in new tab..." target="_blank" rel="noopener"> <a href="{{archive_url}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./{{domain}}</code></p> <p class="card-text"><code>./{{domain}}</code></p>
@ -402,7 +404,7 @@
{% if SAVE_ARCHIVE_DOT_ORG %} {% if SAVE_ARCHIVE_DOT_ORG %}
<div class="col-lg-2"> <div class="col-lg-2">
<div class="card"> <div class="card">
<iframe class="card-img-top" src="{{archive_org_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe> <iframe class="card-img-top" src="{{archive_org_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body"> <div class="card-body">
<a href="{{archive_org_path}}" title="Open in new tab..." target="_blank" rel="noopener"> <a href="{{archive_org_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>🌐 web.archive.org/web/...</code></p> <p class="card-text"><code>🌐 web.archive.org/web/...</code></p>
@ -414,7 +416,7 @@
{% endif %} {% endif %}
<div class="col-lg-2"> <div class="col-lg-2">
<div class="card"> <div class="card">
<iframe class="card-img-top" src="{{url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe> <iframe class="card-img-top" src="{{url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body"> <div class="card-body">
<a href="{{url}}" title="Open in new tab..." target="_blank" rel="noopener"> <a href="{{url}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>🌐 {{domain}}</code></p> <p class="card-text"><code>🌐 {{domain}}</code></p>
@ -425,7 +427,7 @@
</div> </div>
<div class="col-lg-2"> <div class="col-lg-2">
<div class="card"> <div class="card">
<iframe class="card-img-top" src="{{headers_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe> <iframe class="card-img-top" src="{{headers_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body"> <div class="card-body">
<a href="{{headers_path}}" title="Open in new tab..." target="_blank" rel="noopener"> <a href="{{headers_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./headers.json</code></p> <p class="card-text"><code>./headers.json</code></p>
@ -436,7 +438,7 @@
</div> </div>
<div class="col-lg-2"> <div class="col-lg-2">
<div class="card"> <div class="card">
<iframe class="card-img-top" src="{{dom_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe> <iframe class="card-img-top" src="{{dom_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body"> <div class="card-body">
<a href="{{dom_path}}" title="Open in new tab..." target="_blank" rel="noopener"> <a href="{{dom_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./output.html</code></p> <p class="card-text"><code>./output.html</code></p>
@ -447,7 +449,7 @@
</div> </div>
<div class="col-lg-2"> <div class="col-lg-2">
<div class="card"> <div class="card">
<iframe class="card-img-top" src="{{readability_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe> <iframe class="card-img-top" src="{{readability_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body"> <div class="card-body">
<a href="{{readability_path}}" title="Open in new tab..." target="_blank" rel="noopener"> <a href="{{readability_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./readability/content.html</code></p> <p class="card-text"><code>./readability/content.html</code></p>
@ -459,7 +461,7 @@
<br/> <br/>
<div class="col-lg-2"> <div class="col-lg-2">
<div class="card"> <div class="card">
<iframe class="card-img-top" src="{{mercury_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe> <iframe class="card-img-top" src="{{mercury_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body"> <div class="card-body">
<a href="{{mercury_path}}" title="Open in new tab..." target="_blank" rel="noopener"> <a href="{{mercury_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./mercury/content.html</code></p> <p class="card-text"><code>./mercury/content.html</code></p>
@ -470,7 +472,7 @@
</div> </div>
<div class="col-lg-2"> <div class="col-lg-2">
<div class="card"> <div class="card">
<iframe class="card-img-top" src="{{media_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe> <iframe class="card-img-top" src="{{media_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body"> <div class="card-body">
<a href="{{media_path}}" title="Open in new tab..." target="_blank" rel="noopener"> <a href="{{media_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./media/*.mp4</code></p> <p class="card-text"><code>./media/*.mp4</code></p>
@ -481,7 +483,7 @@
</div> </div>
<div class="col-lg-2"> <div class="col-lg-2">
<div class="card"> <div class="card">
<iframe class="card-img-top" src="{{git_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe> <iframe class="card-img-top" src="{{git_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body"> <div class="card-body">
<a href="{{git_path}}" title="Open in new tab..." target="_blank" rel="noopener"> <a href="{{git_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./git/*.git</code></p> <p class="card-text"><code>./git/*.git</code></p>

60
archivebox/templates/static/admin.css
View file

@ -1,3 +1,9 @@
* {
-webkit-box-sizing: border-box;
-moz-box-sizing: border-box;
box-sizing: border-box;
}
#logo { #logo {
height: 30px; height: 30px;
vertical-align: -6px; vertical-align: -6px;
@ -36,6 +42,23 @@ div.breadcrumbs {
padding: 6px 15px; padding: 6px 15px;
} }
#toolbar #searchbar {
height: 25px;
}
#snapshot-view-mode {
float: right;
margin-bottom: -40px;
display: inline-block;
margin-top: 3px;
margin-right: 10px;
font-size: 14px;
opacity: 0.8;
}
#snapshot-view-mode a {
color: #ccc;
}
body.model-snapshot.change-list div.breadcrumbs, body.model-snapshot.change-list div.breadcrumbs,
body.model-snapshot.change-list #content .object-tools { body.model-snapshot.change-list #content .object-tools {
display: none; display: none;
@ -92,6 +115,14 @@ body.model-snapshot.change-list #content .object-tools {
background: none; background: none;
margin-right: 0px; margin-right: 0px;
width: auto; width: auto;
max-height: 40px;
overflow: hidden;
display: block;
}
@media (max-width: 1000px) {
#content #changelist .actions {
max-height: 200px;
}
} }
#content #changelist .actions .button { #content #changelist .actions .button {
@ -116,20 +147,45 @@ body.model-snapshot.change-list #content .object-tools {
background-color:lightseagreen; background-color:lightseagreen;
color: #333; color: #333;
} }
#content #changelist .actions .button[name=resnapshot_snapshot] {
background-color: #9ee54b;
color: #333;
}
#content #changelist .actions .button[name=overwrite_snapshots] { #content #changelist .actions .button[name=overwrite_snapshots] {
background-color: #ffaa31; background-color: #ffaa31;
color: #333; color: #333;
margin-left: 10px;
} }
#content #changelist .actions .button[name=delete_snapshots] { #content #changelist .actions .button[name=delete_snapshots] {
background-color: #f91f74; background-color: #f91f74;
color: rgb(255 248 252 / 64%); color: rgb(255 248 252 / 64%);
} }
#content #changelist .actions .button[name=add_tags] {
}
#content #changelist .actions .button[name=remove_tags] {
margin-right: 25px;
}
#content #changelist .actions .select2-selection {
max-height: 25px;
}
#content #changelist .actions .select2-container--admin-autocomplete.select2-container {
width: auto !important;
min-width: 90px;
}
#content #changelist .actions .select2-selection__rendered .select2-selection__choice {
margin-top: 3px;
}
#content #changelist-filter h2 { #content #changelist-filter h2 {
border-radius: 4px 4px 0px 0px; border-radius: 4px 4px 0px 0px;
} }
#changelist .paginator {
border-top: 0px;
border-bottom: 0px;
}
@media (min-width: 767px) { @media (min-width: 767px) {
#content #changelist-filter { #content #changelist-filter {
top: 35px; top: 35px;
@ -157,7 +213,7 @@ body.model-snapshot.change-list #content .object-tools {
#content a img.favicon { #content a img.favicon {
height: 20px; height: 20px;
width: 20px; max-width: 28px;
vertical-align: -5px; vertical-align: -5px;
padding-right: 6px; padding-right: 6px;
} }
@ -177,7 +233,7 @@ body.model-snapshot.change-list #content .object-tools {
#content th.field-added, #content td.field-updated { #content th.field-added, #content td.field-updated {
word-break: break-word; word-break: break-word;
min-width: 128px; min-width: 135px;
white-space: normal; white-space: normal;
} }

10
archivebox/util.py
View file

@ -11,7 +11,7 @@ from functools import wraps
from hashlib import sha256 from hashlib import sha256
from urllib.parse import urlparse, quote, unquote from urllib.parse import urlparse, quote, unquote
from html import escape, unescape from html import escape, unescape
from datetime import datetime from datetime import datetime, timezone
from dateparser import parse as dateparser from dateparser import parse as dateparser
from requests.exceptions import RequestException, ReadTimeout from requests.exceptions import RequestException, ReadTimeout
@ -51,7 +51,7 @@ htmlencode = lambda s: s and escape(s, quote=True)
htmldecode = lambda s: s and unescape(s) htmldecode = lambda s: s and unescape(s)
short_ts = lambda ts: str(parse_date(ts).timestamp()).split('.')[0] short_ts = lambda ts: str(parse_date(ts).timestamp()).split('.')[0]
ts_to_date = lambda ts: ts and parse_date(ts).strftime('%Y-%m-%d %H:%M') ts_to_date_str = lambda ts: ts and parse_date(ts).strftime('%Y-%m-%d %H:%M')
ts_to_iso = lambda ts: ts and parse_date(ts).isoformat() ts_to_iso = lambda ts: ts and parse_date(ts).isoformat()
@ -144,13 +144,17 @@ def parse_date(date: Any) -> Optional[datetime]:
return None return None
if isinstance(date, datetime): if isinstance(date, datetime):
if date.tzinfo is None:
return date.replace(tzinfo=timezone.utc)
assert date.tzinfo.utcoffset(datetime.now()).seconds == 0, 'Refusing to load a non-UTC date!'
return date return date
if isinstance(date, (float, int)): if isinstance(date, (float, int)):
date = str(date) date = str(date)
if isinstance(date, str): if isinstance(date, str):
return dateparser(date) return dateparser(date, settings={'TIMEZONE': 'UTC'}).replace(tzinfo=timezone.utc)
raise ValueError('Tried to parse invalid date! {}'.format(date)) raise ValueError('Tried to parse invalid date! {}'.format(date))