Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-22 12:13:05 +00:00
Code Issues Projects Releases Packages Wiki Activity

add timezone support, tons of CSS and layout improvements, more detailed snapshot admin form info, ability to sort by recently updated, better grid view styling, better table layouts, better dark mode support

Browse source
This commit is contained in:
Nick Sweeting 2021-04-10 04:19:30 -04:00
parent cf7d7e4990
commit a9986f1f05
28 changed files with 681 additions and 549 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
archivebox/config.py
View file

@ -34,7 +34,7 @@ import django
from hashlib import md5
from pathlib import Path
from datetime import datetime
from datetime import datetime, timezone
from typing import Optional, Type, Tuple, Dict, Union, List
from subprocess import run, PIPE, DEVNULL
from configparser import ConfigParser
@ -80,7 +80,8 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
'PUBLIC_ADD_VIEW': {'type': bool, 'default': False},
'FOOTER_INFO': {'type': str, 'default': 'Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.'},
'SNAPSHOTS_PER_PAGE': {'type': int, 'default': 40},
'CUSTOM_TEMPLATES_DIR': {'type': str, 'default': None}
'CUSTOM_TEMPLATES_DIR': {'type': str, 'default': None},
'TIME_ZONE': {'type': str, 'default': 'UTC'},
},
'ARCHIVE_METHOD_TOGGLES': {
@ -1105,7 +1106,7 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG,
# log startup message to the error log
with open(settings.ERROR_LOG, "a+", encoding='utf-8') as f:
command = ' '.join(sys.argv)
ts = datetime.now().strftime('%Y-%m-%d__%H:%M:%S')
ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S')
f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n")

1
archivebox/core/admin.py
View file

@ -3,6 +3,7 @@ __package__ = 'archivebox.core'
from io import StringIO
from pathlib import Path
from contextlib import redirect_stdout
from datetime import datetime, timezone
from django.contrib import admin
from django.urls import path

17
archivebox/core/settings.py
View file

@ -19,9 +19,9 @@ from ..config import (
SQL_INDEX_FILENAME,
OUTPUT_DIR,
LOGS_DIR,
TIME_ZONE,
)
IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
@ -154,6 +154,7 @@ DATABASES = {
'timeout': 60,
'check_same_thread': False,
},
'TIME_ZONE': 'UTC',
# DB setup is sometimes modified at runtime by setup_django() in config.py
}
}
@ -182,6 +183,7 @@ ALLOWED_HOSTS = ALLOWED_HOSTS.split(',')
SECURE_BROWSER_XSS_FILTER = True
SECURE_CONTENT_TYPE_NOSNIFF = True
SECURE_REFERRER_POLICY = 'strict-origin-when-cross-origin'
CSRF_COOKIE_SECURE = False
SESSION_COOKIE_SECURE = False
@ -217,14 +219,17 @@ if IS_SHELL:
################################################################################
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = False
USE_L10N = False
USE_TZ = False
USE_I18N = True
USE_L10N = True
USE_TZ = True
DATETIME_FORMAT = 'Y-m-d g:iA'
SHORT_DATETIME_FORMAT = 'Y-m-d h:iA'
from django.conf.locale.en import formats as en_formats
en_formats.DATETIME_FORMAT = DATETIME_FORMAT
en_formats.SHORT_DATETIME_FORMAT = SHORT_DATETIME_FORMAT
################################################################################
### Logging Settings

13
archivebox/core/templatetags/core_tags.py
View file

@ -1,22 +1,15 @@
from django import template
from django.urls import reverse
from django.contrib.admin.templatetags.base import InclusionAdminNode
from django.templatetags.static import static
from typing import Union
from core.models import ArchiveResult
register = template.Library()
@register.simple_tag
def snapshot_image(snapshot):
result = ArchiveResult.objects.filter(snapshot=snapshot, extractor='screenshot', status='succeeded').first()
if result:
return reverse('Snapshot', args=[f'{str(snapshot.timestamp)}/{result.output}'])
return static('archive.png')
@register.filter(name='split')
def split(value, separator: str=','):
return (value or '').split(separator)
@register.filter
def file_size(num_bytes: Union[int, float]) -> str:

4
archivebox/extractors/__init__.py
View file

@ -4,7 +4,7 @@ import os
from pathlib import Path
from typing import Optional, List, Iterable, Union
from datetime import datetime
from datetime import datetime, timezone
from django.db.models import QuerySet
from ..index.schema import Link
@ -94,7 +94,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
link = load_link_details(link, out_dir=out_dir)
write_link_details(link, out_dir=out_dir, skip_sql_index=False)
log_link_archiving_started(link, out_dir, is_new)
link = link.overwrite(updated=datetime.now())
link = link.overwrite(updated=datetime.now(timezone.utc))
stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
for method_name, should_run, method_function in ARCHIVE_METHODS:

1
archivebox/extractors/readability.py
View file

@ -92,6 +92,7 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
result = run(cmd, cwd=out_dir, timeout=timeout)
try:
result_json = json.loads(result.stdout)
assert result_json and 'content' in result_json
except json.JSONDecodeError:
raise ArchiveError('Readability was not able to archive the page', result.stdout + result.stderr)

4
archivebox/extractors/wget.py
View file

@ -4,7 +4,7 @@ import re
from pathlib import Path
from typing import Optional
from datetime import datetime
from datetime import datetime, timezone
from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from ..system import run, chmod_file
@ -51,7 +51,7 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
if SAVE_WARC:
warc_dir = out_dir / "warc"
warc_dir.mkdir(exist_ok=True)
warc_path = warc_dir / str(int(datetime.now().timestamp()))
warc_path = warc_dir / str(int(datetime.now(timezone.utc).timestamp()))
# WGET CLI Docs: https://www.gnu.org/software/wget/manual/wget.html
output: ArchiveOutput = None

14
archivebox/index/html.py
View file

@ -1,7 +1,7 @@
__package__ = 'archivebox.index'
from pathlib import Path
from datetime import datetime
from datetime import datetime, timezone
from collections import defaultdict
from typing import List, Optional, Iterator, Mapping
@ -13,7 +13,7 @@ from ..system import atomic_write
from ..logging_util import printable_filesize
from ..util import (
enforce_types,
ts_to_date,
ts_to_date_str,
urlencode,
htmlencode,
urldecode,
@ -62,8 +62,8 @@ def main_index_template(links: List[Link], template: str=MAIN_INDEX_TEMPLATE) ->
'version': VERSION,
'git_sha': VERSION, # not used anymore, but kept for backwards compatibility
'num_links': str(len(links)),
'date_updated': datetime.now().strftime('%Y-%m-%d'),
'time_updated': datetime.now().strftime('%Y-%m-%d %H:%M'),
'date_updated': datetime.now(timezone.utc).strftime('%Y-%m-%d'),
'time_updated': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M'),
'links': [link._asdict(extended=True) for link in links],
'FOOTER_INFO': FOOTER_INFO,
})
@ -103,7 +103,7 @@ def link_details_template(link: Link) -> str:
'size': printable_filesize(link.archive_size) if link.archive_size else 'pending',
'status': 'archived' if link.is_archived else 'not yet archived',
'status_color': 'success' if link.is_archived else 'danger',
'oldest_archive_date': ts_to_date(link.oldest_archive_date),
'oldest_archive_date': ts_to_date_str(link.oldest_archive_date),
'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
})
@ -120,7 +120,7 @@ def snapshot_icons(snapshot) -> str:
def calc_snapshot_icons():
from core.models import EXTRACTORS
# start = datetime.now()
# start = datetime.now(timezone.utc)
archive_results = snapshot.archiveresult_set.filter(status="succeeded", output__isnull=False)
link = snapshot.as_link()
@ -183,7 +183,7 @@ def snapshot_icons(snapshot) -> str:
"archive_org", icons.get("archive_org", "?"))
result = format_html('<span class="files-icons" style="font-size: 1.1em; opacity: 0.8; min-width: 240px; display: inline-block">{}<span>', mark_safe(output))
# end = datetime.now()
# end = datetime.now(timezone.utc)
# print(((end - start).total_seconds()*1000) // 1, 'ms')
return result

4
archivebox/index/json.py
View file

@ -5,7 +5,7 @@ import sys
import json as pyjson
from pathlib import Path
from datetime import datetime
from datetime import datetime, timezone
from typing import List, Optional, Iterator, Any, Union
from .schema import Link
@ -44,7 +44,7 @@ def generate_json_index_from_links(links: List[Link], with_headers: bool):
output = {
**MAIN_INDEX_HEADER,
'num_links': len(links),
'updated': datetime.now(),
'updated': datetime.now(timezone.utc),
'last_run_cmd': sys.argv,
'links': links,
}

17
archivebox/index/schema.py
View file

@ -10,7 +10,7 @@ __package__ = 'archivebox.index'
from pathlib import Path
from datetime import datetime, timedelta
from datetime import datetime, timezone, timedelta
from typing import List, Dict, Any, Optional, Union
@ -19,7 +19,7 @@ from dataclasses import dataclass, asdict, field, fields
from django.utils.functional import cached_property
from ..system import get_dir_size
from ..util import ts_to_date_str, parse_date
from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME
class ArchiveError(Exception):
@ -203,7 +203,7 @@ class Link:
'extension': self.extension,
'is_static': self.is_static,
'tags_str': self.tags, # only used to render static index in index/html.py, remove if no longer needed there
'tags_str': (self.tags or '').strip(','), # only used to render static index in index/html.py, remove if no longer needed there
'icons': None, # only used to render static index in index/html.py, remove if no longer needed there
'bookmarked_date': self.bookmarked_date,
@ -325,13 +325,11 @@ class Link:
### Pretty Printing Helpers
@property
def bookmarked_date(self) -> Optional[str]:
from ..util import ts_to_date
max_ts = (datetime.now() + timedelta(days=30)).timestamp()
max_ts = (datetime.now(timezone.utc) + timedelta(days=30)).timestamp()
if self.timestamp and self.timestamp.replace('.', '').isdigit():
if 0 < float(self.timestamp) < max_ts:
return ts_to_date(datetime.fromtimestamp(float(self.timestamp)))
return ts_to_date_str(datetime.fromtimestamp(float(self.timestamp)))
else:
return str(self.timestamp)
return None
@ -339,13 +337,12 @@ class Link:
@property
def updated_date(self) -> Optional[str]:
from ..util import ts_to_date
return ts_to_date(self.updated) if self.updated else None
return ts_to_date_str(self.updated) if self.updated else None
@property
def archive_dates(self) -> List[datetime]:
return [
result.start_ts
parse_date(result.start_ts)
for method in self.history.keys()
for result in self.history[method]
]

30
archivebox/logging_util.py
View file

@ -10,7 +10,7 @@ from math import log
from multiprocessing import Process
from pathlib import Path
from datetime import datetime
from datetime import datetime, timezone
from dataclasses import dataclass
from typing import Any, Optional, List, Dict, Union, IO, TYPE_CHECKING
@ -138,17 +138,19 @@ class TimedProgress:
"""Show a progress bar and measure elapsed time until .end() is called"""
def __init__(self, seconds, prefix=''):
self.SHOW_PROGRESS = SHOW_PROGRESS
if self.SHOW_PROGRESS:
self.p = Process(target=progress_bar, args=(seconds, prefix))
self.p.start()
self.stats = {'start_ts': datetime.now(), 'end_ts': None}
self.stats = {'start_ts': datetime.now(timezone.utc), 'end_ts': None}
def end(self):
"""immediately end progress, clear the progressbar line, and save end_ts"""
end_ts = datetime.now()
end_ts = datetime.now(timezone.utc)
self.stats['end_ts'] = end_ts
if self.SHOW_PROGRESS:
@ -231,7 +233,7 @@ def progress_bar(seconds: int, prefix: str='') -> None:
def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional[str], pwd: str):
cmd = ' '.join(('archivebox', subcommand, *subcommand_args))
stderr('{black}[i] [{now}] ArchiveBox v{VERSION}: {cmd}{reset}'.format(
now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
now=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
VERSION=VERSION,
cmd=cmd,
**ANSI,
@ -243,7 +245,7 @@ def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional
def log_importing_started(urls: Union[str, List[str]], depth: int, index_only: bool):
_LAST_RUN_STATS.parse_start_ts = datetime.now()
_LAST_RUN_STATS.parse_start_ts = datetime.now(timezone.utc)
print('{green}[+] [{}] Adding {} links to index (crawl depth={}){}...{reset}'.format(
_LAST_RUN_STATS.parse_start_ts.strftime('%Y-%m-%d %H:%M:%S'),
len(urls) if isinstance(urls, list) else len(urls.split('\n')),
@ -256,7 +258,7 @@ def log_source_saved(source_file: str):
print(' > Saved verbatim input to {}/{}'.format(SOURCES_DIR_NAME, source_file.rsplit('/', 1)[-1]))
def log_parsing_finished(num_parsed: int, parser_name: str):
_LAST_RUN_STATS.parse_end_ts = datetime.now()
_LAST_RUN_STATS.parse_end_ts = datetime.now(timezone.utc)
print(' > Parsed {} URLs from input ({})'.format(num_parsed, parser_name))
def log_deduping_finished(num_new_links: int):
@ -270,7 +272,7 @@ def log_crawl_started(new_links):
### Indexing Stage
def log_indexing_process_started(num_links: int):
start_ts = datetime.now()
start_ts = datetime.now(timezone.utc)
_LAST_RUN_STATS.index_start_ts = start_ts
print()
print('{black}[*] [{}] Writing {} links to main index...{reset}'.format(
@ -281,7 +283,7 @@ def log_indexing_process_started(num_links: int):
def log_indexing_process_finished():
end_ts = datetime.now()
end_ts = datetime.now(timezone.utc)
_LAST_RUN_STATS.index_end_ts = end_ts
@ -297,7 +299,8 @@ def log_indexing_finished(out_path: str):
### Archiving Stage
def log_archiving_started(num_links: int, resume: Optional[float]=None):
start_ts = datetime.now()
start_ts = datetime.now(timezone.utc)
_LAST_RUN_STATS.archiving_start_ts = start_ts
print()
if resume:
@ -315,7 +318,8 @@ def log_archiving_started(num_links: int, resume: Optional[float]=None):
))
def log_archiving_paused(num_links: int, idx: int, timestamp: str):
end_ts = datetime.now()
end_ts = datetime.now(timezone.utc)
_LAST_RUN_STATS.archiving_end_ts = end_ts
print()
print('\n{lightyellow}[X] [{now}] Downloading paused on link {timestamp} ({idx}/{total}){reset}'.format(
@ -330,7 +334,8 @@ def log_archiving_paused(num_links: int, idx: int, timestamp: str):
print(' archivebox update --resume={}'.format(timestamp))
def log_archiving_finished(num_links: int):
end_ts = datetime.now()
end_ts = datetime.now(timezone.utc)
_LAST_RUN_STATS.archiving_end_ts = end_ts
assert _LAST_RUN_STATS.archiving_start_ts is not None
seconds = end_ts.timestamp() - _LAST_RUN_STATS.archiving_start_ts.timestamp()
@ -356,6 +361,7 @@ def log_archiving_finished(num_links: int):
def log_link_archiving_started(link: "Link", link_dir: str, is_new: bool):
# [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford"
# http://www.benstopford.com/2015/02/14/log-structured-merge-trees/
# > output/archive/1478739709
@ -363,7 +369,7 @@ def log_link_archiving_started(link: "Link", link_dir: str, is_new: bool):
print('\n[{symbol_color}{symbol}{reset}] [{symbol_color}{now}{reset}] "{title}"'.format(
symbol_color=ANSI['green' if is_new else 'black'],
symbol='+' if is_new else '√',
now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
now=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
title=link.title or link.base_url,
**ANSI,
))

1
archivebox/main.py
View file

@ -585,6 +585,7 @@ def add(urls: Union[str, List[str]],
update_all: bool=not ONLY_NEW,
index_only: bool=False,
overwrite: bool=False,
# duplicate: bool=False, # TODO: reuse the logic from admin.py resnapshot to allow adding multiple snapshots by appending timestamp automatically
init: bool=False,
extractors: str="",
parser: str="auto",

6
archivebox/parsers/__init__.py
View file

@ -11,7 +11,7 @@ import re
from io import StringIO
from typing import IO, Tuple, List, Optional
from datetime import datetime
from datetime import datetime, timezone
from pathlib import Path
from ..system import atomic_write
@ -147,7 +147,7 @@ def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None,
@enforce_types
def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir: Path=OUTPUT_DIR) -> str:
ts = str(datetime.now().timestamp()).split('.', 1)[0]
ts = str(datetime.now(timezone.utc).timestamp()).split('.', 1)[0]
source_path = str(out_dir / SOURCES_DIR_NAME / filename.format(ts=ts))
atomic_write(source_path, raw_text)
log_source_saved(source_file=source_path)
@ -157,7 +157,7 @@ def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir:
@enforce_types
def save_file_as_source(path: str, timeout: int=TIMEOUT, filename: str='{ts}-{basename}.txt', out_dir: Path=OUTPUT_DIR) -> str:
"""download a given url's content into output/sources/domain-<timestamp>.txt"""
ts = str(datetime.now().timestamp()).split('.', 1)[0]
ts = str(datetime.now(timezone.utc).timestamp()).split('.', 1)[0]
source_path = str(OUTPUT_DIR / SOURCES_DIR_NAME / filename.format(basename=basename(path), ts=ts))
if any(path.startswith(s) for s in ('http://', 'https://', 'ftp://')):

4
archivebox/parsers/generic_html.py
View file

@ -4,7 +4,7 @@ __package__ = 'archivebox.parsers'
import re
from typing import IO, Iterable, Optional
from datetime import datetime
from datetime import datetime, timezone
from ..index.schema import Link
from ..util import (
@ -46,7 +46,7 @@ def parse_generic_html_export(html_file: IO[str], root_url: Optional[str]=None,
for archivable_url in re.findall(URL_REGEX, url):
yield Link(
url=htmldecode(archivable_url),
timestamp=str(datetime.now().timestamp()),
timestamp=str(datetime.now(timezone.utc).timestamp()),
title=None,
tags=None,
sources=[html_file.name],

4
archivebox/parsers/generic_json.py
View file

@ -3,7 +3,7 @@ __package__ = 'archivebox.parsers'
import json
from typing import IO, Iterable
from datetime import datetime
from datetime import datetime, timezone
from ..index.schema import Link
from ..util import (
@ -30,7 +30,7 @@ def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
raise Exception('JSON must contain URL in each entry [{"url": "http://...", ...}, ...]')
# Parse the timestamp
ts_str = str(datetime.now().timestamp())
ts_str = str(datetime.now(timezone.utc).timestamp())
if link.get('timestamp'):
# chrome/ff histories use a very precise timestamp
ts_str = str(link['timestamp'] / 10000000)

8
archivebox/parsers/generic_txt.py
View file

@ -4,7 +4,7 @@ __description__ = 'Plain Text'
import re
from typing import IO, Iterable
from datetime import datetime
from datetime import datetime, timezone
from pathlib import Path
from ..index.schema import Link
@ -29,7 +29,7 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
if Path(line).exists():
yield Link(
url=line,
timestamp=str(datetime.now().timestamp()),
timestamp=str(datetime.now(timezone.utc).timestamp()),
title=None,
tags=None,
sources=[text_file.name],
@ -42,7 +42,7 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
for url in re.findall(URL_REGEX, line):
yield Link(
url=htmldecode(url),
timestamp=str(datetime.now().timestamp()),
timestamp=str(datetime.now(timezone.utc).timestamp()),
title=None,
tags=None,
sources=[text_file.name],
@ -54,7 +54,7 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
for sub_url in re.findall(URL_REGEX, line[1:]):
yield Link(
url=htmldecode(sub_url),
timestamp=str(datetime.now().timestamp()),
timestamp=str(datetime.now(timezone.utc).timestamp()),
title=None,
tags=None,
sources=[text_file.name],

4
archivebox/parsers/pinboard_rss.py
View file

@ -2,7 +2,7 @@ __package__ = 'archivebox.parsers'
from typing import IO, Iterable
from datetime import datetime
from datetime import datetime, timezone
from xml.etree import ElementTree
@ -36,7 +36,7 @@ def parse_pinboard_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
if ts_str:
time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
else:
time = datetime.now()
time = datetime.now(timezone.utc)
yield Link(
url=htmldecode(url),

4
archivebox/parsers/url_list.py
View file

@ -4,7 +4,7 @@ __description__ = 'URL list'
import re
from typing import IO, Iterable
from datetime import datetime
from datetime import datetime, timezone
from ..index.schema import Link
from ..util import (
@ -25,7 +25,7 @@ def parse_url_list(text_file: IO[str], **_kwargs) -> Iterable[Link]:
yield Link(
url=url,
timestamp=str(datetime.now().timestamp()),
timestamp=str(datetime.now(timezone.utc).timestamp()),
title=None,
tags=None,
sources=[text_file.name],

292
archivebox/templates/admin/base.html
View file

@ -4,78 +4,69 @@
<!DOCTYPE html>
<html lang="{{ LANGUAGE_CODE|default:"en-us" }}" {% if LANGUAGE_BIDI %}dir="rtl"{% endif %}>
<head>
<title>{% block title %}{% endblock %} | ArchiveBox</title>
<link rel="stylesheet" type="text/css" href="{% block stylesheet %}{% static "admin/css/base.css" %}{% endblock %}">
{% block extrastyle %}{% endblock %}
{% if LANGUAGE_BIDI %}<link rel="stylesheet" type="text/css" href="{% block stylesheet_rtl %}{% static "admin/css/rtl.css" %}{% endblock %}">{% endif %}
{% block extrahead %}{% endblock %}
{% block responsive %}
<head>
<title>{% block title %}Home{% endblock %} | ArchiveBox</title>
{% block blockbots %}
<meta name="robots" content="NONE,NOARCHIVE">
{% endblock %}
<link rel="stylesheet" type="text/css" href="{% block stylesheet %}{% static "admin/css/base.css" %}{% endblock %}">
{% block extrastyle %}{% endblock %}
{% if LANGUAGE_BIDI %}
<link rel="stylesheet" type="text/css" href="{% block stylesheet_rtl %}{% static "admin/css/rtl.css" %}{% endblock %}">
{% endif %}
{% block responsive %}
<meta name="viewport" content="user-scalable=no, width=device-width, initial-scale=1.0, maximum-scale=1.0">
<link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive.css" %}">
{% if LANGUAGE_BIDI %}<link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive_rtl.css" %}">{% endif %}
{% endblock %}
{% block blockbots %}<meta name="robots" content="NONE,NOARCHIVE">{% endblock %}
<link rel="stylesheet" type="text/css" href="{% static "admin.css" %}">
</head>
{% load i18n %}
{% if LANGUAGE_BIDI %}
<link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive_rtl.css" %}">
{% endif %}
{% endblock %}
<body class="{% if is_popup %}popup {% endif %}{% block bodyclass %}{% endblock %}"
data-admin-utc-offset="{% now "Z" %}">
<style>
/* Loading Progress Bar */
#progress {
position: absolute;
z-index: 1000;
top: 0px;
left: -6px;
width: 2%;
opacity: 1;
height: 2px;
background: #1a1a1a;
border-radius: 1px;
transition: width 4s ease-out, opacity 400ms linear;
}
@-moz-keyframes bugfix { from { padding-right: 1px ; } to { padding-right: 0; } }
</style>
<link rel="stylesheet" type="text/css" href="{% static "admin.css" %}">
<script>
// Page Loading Bar
window.loadStart = function(distance) {
var distance = distance || 0;
// only add progrstess bar if not already present
if (django.jQuery("#loading-bar").length == 0) {
django.jQuery("body").add("<div id=\"loading-bar\"></div>");
function selectSnapshotListView(e) {
e && e.stopPropagation()
e && e.preventDefault()
console.log('Switching to Snapshot list view...')
localStorage.setItem('preferred_snapshot_view_mode', 'list')
window.location = "{% url 'admin:core_snapshot_changelist' %}" + document.location.search
return false
}
if (django.jQuery("#progress").length === 0) {
django.jQuery("body").append(django.jQuery("<div></div>").attr("id", "progress"));
let last_distance = (distance || (30 + (Math.random() * 30)))
django.jQuery("#progress").width(last_distance + "%");
setInterval(function() {
last_distance += Math.random()
django.jQuery("#progress").width(last_distance + "%");
}, 1000)
function selectSnapshotGridView(e) {
e && e.stopPropagation()
e && e.preventDefault()
console.log('Switching to Snapshot grid view...')
localStorage.setItem('preferred_snapshot_view_mode', 'grid')
window.location = "{% url 'admin:grid' %}" + document.location.search
return false
}
};
window.loadFinish = function() {
django.jQuery("#progress").width("101%").delay(200).fadeOut(400, function() {
django.jQuery(this).remove();
});
};
window.loadStart();
window.addEventListener('beforeunload', function() {window.loadStart(27)});
document.addEventListener('DOMContentLoaded', function() {window.loadFinish()});
const preferred_view = localStorage.getItem('preferred_snapshot_view_mode') || 'unset'
const current_view = (
window.location.pathname === "{% url 'admin:core_snapshot_changelist' %}"
? 'list'
: 'grid')
console.log('Preferred snapshot view is:', preferred_view, 'Current view mode is:', current_view)
if (preferred_view === 'grid' && current_view !== 'grid') {
selectSnapshotGridView()
}
</script>
{% block extrahead %}{% endblock %}
</head>
<!-- Container -->
<div id="container">
<body class="{% if is_popup %}popup {% endif %}{% block bodyclass %}{% endblock %}" data-admin-utc-offset="{% now "Z" %}">
{% include 'progressbar.html' %}
<div id="container">
{% if not is_popup %}
<!-- Header -->
<div id="header">
<div id="branding">
<h1 id="site-name">
@ -83,11 +74,6 @@
<img src="{% static 'archive.png' %}" id="logo">
ArchiveBox
</a>
&nbsp; &nbsp;
<small style="display: inline-block;margin-top: 2px;font-size: 18px;opacity: 0.8;">
<a><span id="snapshotListView" style="cursor: pointer">☰</span></a> |
<a><span id="snapshotGridView"style="letter-spacing: -.4em; cursor: pointer;">⣿⣿</span></a>
</small>
</h1>
</div>
{% block usertools %}
@ -97,7 +83,7 @@
{% endblock %}
{% block nav-global %}{% endblock %}
</div>
<!-- END Header -->
{% block breadcrumbs %}
<div class="breadcrumbs">
<a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
@ -108,14 +94,21 @@
{% block messages %}
{% if messages %}
<ul class="messagelist">{% for message in messages %}
<ul class="messagelist">
{% for message in messages %}
<li{% if message.tags %} class="{{ message.tags }}"{% endif %}>{{ message|capfirst }}</li>
{% endfor %}</ul>
{% endfor %}
</ul>
{% endif %}
{% endblock messages %}
<!-- Content -->
<div id="content" class="{% block coltype %}colM{% endblock %}">
{% if opts.model_name == 'snapshot' and cl %}
<small id="snapshot-view-mode">
<a href="#list" title="List view" id="snapshot-view-list">☰</a> |
<a href="#grid" title="Grid view" id="snapshot-view-grid" style="letter-spacing: -.4em;">⣿⣿</a>
</small>
{% endif %}
{% block pretitle %}{% endblock %}
{% block content_title %}{# {% if title %}<h1>{{ title }}</h1>{% endif %} #}{% endblock %}
{% block content %}
@ -125,107 +118,114 @@
{% block sidebar %}{% endblock %}
<br class="clear">
</div>
<!-- END Content -->
{% block footer %}<div id="footer"></div>{% endblock %}
</div>
<!-- END Container -->
</div>
<script>
(function ($) {
<script>
$ = django.jQuery;
$.fn.reverse = [].reverse;
// hide images that fail to load
document.querySelector('body').addEventListener('error', function (e) {
e.target.style.opacity = 0;
}, true)
// setup timezone
{% get_current_timezone as TIME_ZONE %}
window.TIME_ZONE = '{{TIME_ZONE}}'
window.setCookie = function(name, value, days) {
let expires = ""
if (days) {
const date = new Date()
date.setTime(date.getTime() + (days*24*60*60*1000))
expires = "; expires=" + date.toUTCString()
}
document.cookie = name + "=" + (value || "") + expires + "; path=/"
}
function setTimeOffset() {
if (window.GMT_OFFSET) return
window.GMT_OFFSET = -(new Date).getTimezoneOffset()
window.setCookie('GMT_OFFSET', window.GMT_OFFSET, 365)
}
// change the admin actions button from a dropdown to buttons across
function fix_actions() {
var container = $('div.actions');
const container = $('div.actions')
if (container.find('select[name=action] option').length < 10) {
container.find('label:nth-child(1), button[value=0]').hide();
// too many actions to turn into buttons
if (container.find('select[name=action] option').length >= 11) return
var buttons = $('<div></div>')
.appendTo(container)
// hide the empty default option thats just a placeholder with no value
container.find('label:nth-child(1), button[value=0]').hide()
const buttons = $('<div></div>')
.insertAfter('div.actions button[type=submit]')
.css('display', 'inline')
.addClass('class', 'action-buttons');
container.find('select[name=action] option:gt(0)').reverse().each(function () {
const name = this.value
// for each action in the dropdown, turn it into a button instead
container.find('select[name=action] option:gt(0)').each(function () {
const action_type = this.value
$('<button>')
.appendTo(buttons)
.attr('name', this.value)
.attr('type', 'button')
.attr('name', action_type)
.addClass('button')
.text(this.text)
.click(function (e) {
e.preventDefault()
e.stopPropagation()
container.find('select')
.find(':selected').removeAttr('selected').end()
.find('[value=' + name + ']').attr('selected', 'selected').click();
$('#changelist-form button[name="index"]').click();
const num_selected = document.querySelector('.action-counter').innerText.split(' ')[0]
if (action_type === 'overwrite_snapshots') {
const message = (
'Are you sure you want to re-archive (overwrite) ' + num_selected + ' Snapshots?\n\n' +
'This will delete all previously saved files from these Snapshots and re-archive them from scratch.\n\n'
)
if (!window.confirm(message)) return false
}
if (action_type === 'delete_snapshots') {
const message = (
'Are you sure you want to permanently delete ' + num_selected + ' Snapshots?\n\n' +
'They will be removed from your index, and all their Snapshot content on disk will be permanently deleted.'
)
if (!window.confirm(message)) return false
}
// select the action button from the dropdown
container.find('select[name=action]')
.find('op:selected').removeAttr('selected').end()
.find('[value=' + action_type + ']').attr('selected', 'selected').click()
// click submit & replace the archivebox logo with a spinner
$('#changelist-form button[name="index"]').click()
document.querySelector('#logo').outerHTML = '<div class="loader"></div>'
return false
});
});
}
};
function redirectWithQuery(uri){
uri_query = uri + document.location.search;
window.location = uri_query;
};
function selectSnapshotListView(){
localStorage.setItem('currentSnapshotView', 'List');
redirectWithQuery("{% url 'admin:core_snapshot_changelist' %}");
};
function selectSnapshotGridView(){
localStorage.setItem('currentSnapshotView', 'Grid');
redirectWithQuery("{% url 'admin:grid' %}");
};
function setPreferredSnapshotView(view){
urlPath = window.location.pathname;
if((view==="Grid") && urlPath == "{% url 'admin:core_snapshot_changelist' %}"){
selectSnapshotGridView();
})
.appendTo(buttons)
})
console.log('Converted', buttons.children().length, 'admin actions from dropdown to buttons')
}
{% comment %}
else if((view==="List") && urlPath == "{% url 'admin:grid' %}"){
selectSnapshotListView();
function setupSnapshotGridListToggle() {
$("#snapshot-view-list").click(selectSnapshotListView)
$("#snapshot-view-grid").click(selectSnapshotGridView)
}
{% endcomment %}
};
function setupSnapshotViews() {
const preferredSnapshotView = localStorage.getItem('currentSnapshotView');
setPreferredSnapshotView(preferredSnapshotView);
$( document ).ready(function() {
$("#snapshotListView").click(function() {
selectSnapshotListView();
});
$("#snapshotGridView").click(function() {
selectSnapshotGridView();
});
$('input:checkbox').change(function(){
if($(this).is(':checked'))
$(this).parent().parent().parent().parent().addClass('selected-card');
$('#changelist-form .card input:checkbox').change(function() {
if ($(this).is(':checked'))
$(this).parents('.card').addClass('selected-card')
else
$(this).parent().parent().parent().parent().removeClass('selected-card')
});
});
$(this).parents('.card').removeClass('selected-card')
})
};
$(function () {
fix_actions();
setupSnapshotViews();
});
})(django.jQuery);
</script>
</body>
$(document).ready(function() {
fix_actions()
setupSnapshotGridListToggle()
setTimeOffset()
})
</script>
</body>
</html>

289
archivebox/templates/admin/snapshots_grid.html
View file

@ -3,125 +3,140 @@
{% block extrastyle %}
<style>
* {
-webkit-box-sizing: border-box;
-moz-box-sizing: border-box;
box-sizing: border-box;
}
a {
text-decoration: none;
color: orange;
}
h2 {
color: #000;
margin: 2rem 0 .5rem;
font-size: 1.25rem;
font-weight: 400;
{% comment %} text-transform: uppercase; {% endcomment %}
}
card.img {
display: block;
border: 0;
width: 100%;
height: auto;
}
/*************************** Cards *******************************/
.cards {
#changelist-search #searchbar {
min-height: 24px;
}
.cards {
padding-top: 10px;
display: grid;
grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); /* see notes below */
grid-auto-rows: minmax(200px, auto);
grid-gap: 1rem;
}
grid-gap: 14px 14px;
}
.card {
/*height: 200px;*/
/*background: red;*/
border: 2px solid #e7e7e7;
border-radius: 4px;
-webkit-box-shadow: 0 2px 2px rgba(0, 0, 0, 0.15);
box-shadow: 0 2px 2px rgba(0, 0, 0, 0.15);
display: flex;
/* -webkit-box-orient: vertical; */
/* -webkit-box-direction: normal; */
-ms-flex-direction: column;
flex-direction: column;
.cards .card {
position: relative;
max-height: 380px;
overflow: hidden;
background-color: #fffcfc;
border: 1px solid #f1efef;
border-radius: 4px;
box-shadow: 4px 4px 2px 2px rgba(0, 0, 0, 0.01);
text-align: center;
color: #5d5e5e;
} /* li item */
}
.thumbnail img {
height: 100%;
box-sizing: border-box;
max-width: 100%;
max-height: 100%;
.cards .card.selected-card {
border: 3px solid #2196f3;
box-shadow: 2px 3px 6px 2px rgba(0, 0, 221, 0.14);
}
.cards .card .card-thumbnail {
display: block;
width: 100%;
}
height: 345px;
overflow: hidden;
border-radius: 4px;
background-color: #fffcfc;
}
.card-content {
font-size: .75rem;
padding: .5rem;
display: flex;
-webkit-box-orient: vertical;
-webkit-box-direction: normal;
-ms-flex-direction: column;
flex-direction: column;
-webkit-box-flex: 1;
-ms-flex: 1;
flex: 1;
.cards .card .card-thumbnail img {
width: 100%;
height: auto;
border: 0;
}
.cards .card .card-thumbnail.missing img {
opacity: 0.03;
width: 20%;
height: auto;
margin-top: 84px;
}
}
.cards .card .card-tags {
width: 100%;
}
.cards .card .card-tags span {
display: inline-block;
padding: 2px 5px;
border-radius: 5px;
opacity: 0.95;
background-color: #bfdfff;
color: #679ac2;
font-size: 12px;
margin-bottom: 3px;
}
.card-content h4{
vertical-align:bottom;
margin: 1.2em 0 0em 0;
}
.category {
font-size: .75rem;
text-transform: uppercase;
}
.category {
.cards .card .card-footer {
width: 100%;
position: absolute;
top: 5%;
right: 0;
color: #fff;
background: #e74c3c;
padding: 10px 15px;
font-size: 10px;
bottom: 0;
text-align: center;
}
.cards .card .card-title {
padding: 4px 5px;
background-color: #fffcfc;
/*height: 50px;
vertical-align: middle;
line-height: 50px;*/
}
.cards .card .card-title h4 {
color: initial;
display: block;
vertical-align: middle;
line-height: normal;
margin: 0px;
padding: 5px 0px;
font-size: 13.5px;
font-weight: 400;
word-break: break-all;
overflow: hidden;
text-overflow: ellipsis;
max-height: 46px;
}
.cards .card .card-title h4 .title-text {
user-select: all;
}
.cards .card .card-title .link-favicon {
height: 15px;
margin: 2px;
vertical-align: -5px;
display: inline-block;
}
.cards .card .card-info {
padding: 2px 4px;
/*border-top: 1px solid #ddd;*/
background-color: #fffcfc;
font-size: 11px;
color: #333;
}
.cards .card .card-info input[type=checkbox] {
float: right;
width: 18px;
height: 18px;
}
.cards .card .card-info label {
display: inline-block;
height: 20px;
width: 145px;
font-size: 11px;
}
.cards .card .card-info .timestamp {
font-weight: 600;
text-transform: uppercase;
}
.category__01 {
background-color: #50c6db;
}
.tags{
opacity: 0.8;
}
footer {
border-top: 2px solid #e7e7e7;
{% comment %} margin: .5rem 0 0; {% endcomment %}
{% comment %} min-height: 30px; {% endcomment %}
font-size: .5rem;
}
.post-meta {
padding: .3rem;
}
.comments {
margin-left: .5rem;
}
.selected-card{
border: 5px solid #ffaa31;
}
}
.cards .card .card-footer code {
display: inline-block;
width: 100%;
margin-top: 2px;
font-size: 10px;
opacity: 0.4;
user-select: all;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
</style>
@ -130,33 +145,47 @@ footer {
{% block content %}
<section class="cards">
{% for obj in results %}
<article class="card">
<picture class="thumbnail">
<a href="/{{obj.archive_path}}/index.html">
<img class="category__01" src="{% snapshot_image obj%}" alt="" />
</a>
</picture>
<div class="card-content">
{% if obj.tags_str %}
<p class="category category__01 tags">{{obj.tags_str}}</p>
{% endif %}
{% if obj.title %}
<div class="card">
<div class="card-info">
<a href="{% url 'admin:core_snapshot_change' obj.id %}">
<h4>{{obj.title|truncatechars:55 }}</h4>
<span class="timestamp">{{obj.added}}</span>
</a>
{% endif %}
{% comment %} <p> TEXT If needed.</p> {% endcomment %}
</div><!-- .card-content -->
<footer>
<div class="post-meta">
<span style="float:right;"><input type="checkbox" name="_selected_action" value="{{obj.pk}}" class="action-select"></span>
<span class="timestamp">&#128337 {{obj.added}}</span>
<span class="comments">📖{{obj.num_outputs}}</span>
<span>🗄️{{ obj.archive_size | file_size }}</span>
&nbsp; &nbsp;
<label>
<span class="num_outputs">📄 &nbsp; {{obj.num_outputs}}</span> &nbsp; &nbsp;
<span>🗄&nbsp; {{ obj.archive_size | file_size }}</span>
<input type="checkbox" name="_selected_action" value="{{obj.pk}}"/>
</label>
</div>
<a href="/{{obj.archive_path}}/index.html" class="card-thumbnail {% if not obj.thumbnail_url %}missing{% endif %}">
<img src="{{obj.thumbnail_url|default:'/static/spinner.gif' }}" alt="{{obj.title|default:'Not yet archived...'}}" />
</a>
<div class="card-footer">
{% if obj.tags_str %}
<div class="card-tags">
{% for tag in obj.tags_str|split:',' %}
<span>
{{tag}}
</span>
{% endfor %}
</div>
{% endif %}
<div class="card-title" title="{{obj.title}}">
<a href="/{{obj.archive_path}}/index.html">
<h4>
{% if obj.is_archived %}
<img src="/{{obj.archive_path}}/favicon.ico" onerror="this.style.display='none'" class="link-favicon" decoding="async"/>
{% else %}
<img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async"/>
{% endif %}
<span class="title-text">{{obj.title|default:'Pending...' }}</span>
</h4>
</a>
<code title="{{obj.url}}">{{obj.url}}</code>
</div>
</div>
</div>
</footer>
</article>
{% endfor %}
</section>
<br/>
{% endblock %}

2
archivebox/templates/core/add.html
View file

@ -49,7 +49,7 @@
<script>
document.getElementById('add-form').addEventListener('submit', function(event) {
setTimeout(function() {
document.getElementById('add-form').innerHTML = '<center><h3>Adding URLs to index and running archive methods...<h3><br/><div class="loader"></div><br/>Check server log or <a href="/admin/core/archiveresult/?o=-1">Outputs page</a> for progress...</center>'
document.getElementById('add-form').innerHTML = '<center><h3>Adding URLs to index and running archive methods...<h3><br/><div class="loader"></div><br/>Check the server log or the <a href="/admin/core/archiveresult/?o=-1">Log</a> page for progress...</center>'
document.getElementById('delay-warning').style.display = 'block'
}, 200)
return true

27
archivebox/templates/core/base.html
View file

@ -2,35 +2,17 @@
<!DOCTYPE html>
<html lang="en">
<head>
<head>
<title>Archived Sites</title>
<meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="{% static 'admin/css/base.css' %}">
<link rel="stylesheet" type="text/css" href="{% static 'admin.css' %}">
<link rel="stylesheet" href="{% static 'admin.css' %}">
<link rel="stylesheet" href="{% static 'bootstrap.min.css' %}">
<link rel="stylesheet" href="{% static 'jquery.dataTables.min.css' %}" />
<script src="{% static 'jquery.min.js' %}"></script>
{% block extra_head %}
{% endblock %}
<script src="{% static 'jquery.min.js' %}"></script>
<script src="{% static 'jquery.dataTables.min.js' %}"></script>
<script>
document.addEventListener('error', function (e) {
e.target.style.opacity = 0;
}, true)
jQuery(document).ready(function () {
jQuery('#table-bookmarks').DataTable({
searching: false,
paging: false,
stateSave: true, // save state (filtered input, number of entries shown, etc) in localStorage
dom: '<lf<t>ip>', // how to show the table and its helpers (filter, etc) in the DOM
order: [[0, 'desc']],
iDisplayLength: 100,
});
});
</script>
<base href="{% url 'Home' %}">
</head>
<body>
<div id="container">
@ -48,6 +30,7 @@
</div>
<div id="content" class="flex">
{% block body %}
{% endblock %}
</div>
{% block footer %}

41
archivebox/templates/core/index_row.html
View file

@ -1,37 +1,44 @@
{% load static %}
{% load static tz core_tags %}
<tr>
<td title="{{link.timestamp}}"> {% if link.bookmarked_date %} {{ link.bookmarked_date }} {% else %} {{ link.added }} {% endif %} </td>
<td class="title-col" style="opacity: {% if link.title %}1{% else %}0.3{% endif %}">
<td title="Bookmarked: {{link.bookmarked_date|localtime}} ({{link.timestamp}})" data-sort="{{link.added.timestamp}}">
{{ link.added|localtime }}
</td>
<td class="title-col" style="opacity: {% if link.title %}1{% else %}0.3{% endif %}" title="{{link.title|default:'Not yet archived...'}}">
{% if link.is_archived %}
<a href="archive/{{link.timestamp}}/index.html"><img src="archive/{{link.timestamp}}/favicon.ico" onerror="this.style.display='none'" class="link-favicon" decoding="async"></a>
<a href="/archive/{{link.timestamp}}/index.html"><img src="archive/{{link.timestamp}}/favicon.ico" onerror="this.style.display='none'" class="link-favicon" decoding="async"></a>
{% else %}
<a href="archive/{{link.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async" style="height: 15px"></a>
<a href="/archive/{{link.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async" style="height: 15px"></a>
{% endif %}
<a href="archive/{{link.timestamp}}/index.html" title="{{link.title|default:'Not yet archived...'}}">
<span data-title-for="{{link.url}}" data-archived="{{link.is_archived}}">{{link.title|default:'Loading...'|truncatechars:128}}</span>
{% if link.tags_str %}
<span class="tags" style="float: right; border-radius: 5px; background-color: #bfdfff; padding: 2px 5px; margin-left: 4px; margin-top: 1px;">
{% if link.tags_str != None %}
{{link.tags_str|default:''}}
{% else %}
{{ link.tags|default:'' }}
{% endif %}
<a href="/archive/{{link.timestamp}}/index.html" title="{{link.title|default:'Not yet archived...'}}">
<span data-title-for="{{link.url}}" data-archived="{{link.is_archived}}">
{{link.title|default:'Loading...'|truncatechars:128}}
</span>
{% if link.tags_str %}
{% for tag in link.tags_str|split:',' %}
<span class="tag" style="float: right; border-radius: 5px; background-color: #bfdfff; padding: 2px 5px; margin-left: 4px; margin-top: 1px;">
{{tag}}
</span>
{% endfor %}
{% endif %}
</a>
</td>
<td>
<span data-number-for="{{link.url}}" title="Fetching any missing files...">
{% if link.icons %}
{{link.icons}} <small style="float:right; opacity: 0.5">{{link.num_outputs}}</small>
{{link.icons}}&nbsp; <small style="float:right; opacity: 0.5">{{link.num_outputs}}</small>
{% else %}
<a href="archive/{{link.timestamp}}/index.html">📄
<a href="/archive/{{link.timestamp}}/index.html">
📄 &nbsp;
{{link.num_outputs}} <img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="files-spinner" decoding="async" style="height: 15px"/>
</a>
{% endif %}
</span>
</td>
<td style="text-align:left; word-wrap: anywhere;"><a href="{{link.url}}">{{link.url|truncatechars:128}}</a></td>
<td style="text-align:left; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; title="{{link.url}}">
<a href="{{link.url}}">
{{link.url}}
</a>
</td>
</tr>

45
archivebox/templates/core/progressbar.html Normal file
View file

@ -0,0 +1,45 @@
<style>
/* Loading Progress Bar */
#progress {
position: absolute;
z-index: 1000;
top: 0px;
left: -6px;
width: 2%;
opacity: 1;
height: 2px;
background: #1a1a1a;
border-radius: 1px;
transition: width 4s ease-out, opacity 400ms linear;
}
@-moz-keyframes bugfix { from { padding-right: 1px ; } to { padding-right: 0; } }
</style>
<script>
// Page Loading Bar
window.loadStart = function(distance) {
var distance = distance || 0;
// only add progrstess bar if not already present
if (django.jQuery("#loading-bar").length == 0) {
django.jQuery("body").add("<div id=\"loading-bar\"></div>");
}
if (django.jQuery("#progress").length === 0) {
django.jQuery("body").append(django.jQuery("<div></div>").attr("id", "progress"));
let last_distance = (distance || (30 + (Math.random() * 30)))
django.jQuery("#progress").width(last_distance + "%");
setInterval(function() {
last_distance += Math.random()
django.jQuery("#progress").width(last_distance + "%");
}, 1000)
}
};
window.loadFinish = function() {
django.jQuery("#progress").width("101%").delay(200).fadeOut(400, function() {
django.jQuery(this).remove();
});
};
window.loadStart();
window.addEventListener('beforeunload', function() {window.loadStart(27)});
document.addEventListener('DOMContentLoaded', function() {window.loadFinish()});
</script>

27
archivebox/templates/core/public_index.html
View file

@ -1,12 +1,7 @@
{% extends "base.html" %}
{% load static %}
{% load static tz %}
{% block body %}
<style>
#table-bookmarks_info {
display: none;
}
</style>
<div id="toolbar">
<form id="changelist-search" action="{% url 'public-index' %}" method="get">
<div>
@ -19,16 +14,22 @@
onclick="location.href='{% url 'public-index' %}'"
style="background-color: rgba(121, 174, 200, 0.8); height: 30px; font-size: 0.8em; margin-top: 12px; padding-top: 6px; float:right">
</input>
&nbsp;
&nbsp;
{{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
&nbsp;
(Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }})
</div>
</form>
</div>
<table id="table-bookmarks">
<div style="width: 100%; overflow-x: auto;">
<table id="table-bookmarks" style="width: 100%; table-layout: fixed">
<thead>
<tr>
<th style="width: 100px;">Bookmarked</th>
<th style="width: 26vw;">Snapshot ({{page_obj.paginator.count}})</th>
<th style="width: 140px">Files</th>
<th style="width: 16vw;whitespace:nowrap;overflow-x:hidden;">Original URL</th>
<th style="width: 130px">Bookmarked</th>
<th>Snapshot ({{page_obj.paginator.count}})</th>
<th style="width: 258px">Files</th>
<th>Original URL</th>
</tr>
</thead>
<tbody>
@ -37,8 +38,9 @@
{% endfor %}
</tbody>
</table>
<center>
</div>
<br/>
<center>
Showing {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
<br/>
<span class="step-links">
@ -58,7 +60,6 @@
<a href="{% url 'public-index' %}?page={{ page_obj.paginator.num_pages }}">last &raquo;</a>
{% endif %}
</span>
</span>
<br>
</center>
{% endblock %}

118
archivebox/templates/core/snapshot.html
View file

@ -1,3 +1,5 @@
{% load tz core_tags %}
<!DOCTYPE html>
<html lang="en">
<head>
@ -20,7 +22,6 @@
}
header {
background-color: #aa1e55;
padding-bottom: 12px;
}
small {
font-weight: 200;
@ -34,15 +35,15 @@
min-height: 40px;
margin: 0px;
text-align: center;
color: white;
font-size: calc(11px + 0.84vw);
color: #f6f6f6;
font-size: calc(10px + 0.84vw);
font-weight: 200;
padding: 4px 4px;
padding: 3px 4px;
background-color: #aa1e55;
}
.nav > div {
min-height: 30px;
line-height: 1.3;
line-height: 1.2;
}
.header-top a {
text-decoration: none;
@ -68,9 +69,14 @@
.header-archivebox img:hover {
opacity: 0.5;
}
.header-url small {
header small code {
white-space: nowrap;
font-weight: 200;
display: block;
margin-top: -1px;
font-size: 13px;
opacity: 0.8;
user-select: all;
}
.header-url img {
height: 20px;
@ -90,28 +96,38 @@
.info-row .alert {
margin-bottom: 0px;
}
.row.header-bottom {
margin-left: -10px;
margin-right: -10px;
}
.header-bottom .col-lg-2 {
padding-left: 4px;
padding-right: 4px;
}
.header-bottom-frames .card {
overflow: hidden;
box-shadow: 2px 3px 14px 0px rgba(0,0,0,0.02);
margin-top: 10px;
margin-bottom: 5px;
border: 1px solid rgba(0,0,0,3);
border-radius: 14px;
border-radius: 10px;
background-color: black;
overflow: hidden;
}
.card h4 {
font-size: 1.4vw;
}
.card-body {
font-size: 15px;
font-size: 14px;
padding: 13px 10px;
padding-bottom: 6px;
padding-bottom: 1px;
/* padding-left: 3px; */
/* padding-right: 3px; */
/* padding-bottom: 3px; */
line-height: 1.1;
line-height: 1;
word-wrap: break-word;
max-height: 102px;
overflow: hidden;
text-overflow: ellipsis;
background-color: #1a1a1a;
color: #d3d3d3;
}
@ -146,22 +162,12 @@
border-top: 3px solid #aa1e55;
}
.card.selected-card {
border: 1px solid orange;
border: 2px solid orange;
box-shadow: 0px -6px 13px 1px rgba(0,0,0,0.05);
}
.iframe-large {
height: calc(100% - 40px);
}
.pdf-frame {
transform: none;
width: 100%;
height: 160px;
margin-top: -60px;
margin-bottom: 0px;
transform: scale(1.1);
width: 100%;
margin-left: -10%;
}
img.external {
height: 30px;
margin-right: -10px;
@ -185,7 +191,7 @@
}
.header-bottom {
border-top: 1px solid rgba(170, 30, 85, 0.9);
padding-bottom: 12px;
padding-bottom: 1px;
border-bottom: 5px solid rgb(170, 30, 85);
margin-bottom: -1px;
@ -215,10 +221,11 @@
}
.info-chunk {
width: auto;
display:inline-block;
display: inline-block;
text-align: center;
margin: 10px 10px;
margin: 8px 4px;
vertical-align: top;
font-size: 14px;
}
.info-chunk .badge {
margin-top: 5px;
@ -226,13 +233,12 @@
.header-bottom-frames .card-title {
width: 100%;
text-align: center;
font-size: 18px;
margin-bottom: 5px;
font-size: 17px;
margin-bottom: 0px;
display: inline-block;
color: #d3d3d3;
font-weight: 200;
vertical-align: 0px;
margin-top: -6px;
vertical-align: 3px;
}
.header-bottom-frames .card-text {
width: 100%;
@ -277,8 +283,7 @@
<header>
<div class="header-top container-fluid">
<div class="row nav">
<div class="col-lg-2" style="line-height: 64px;">
<div class="col-lg-2" style="line-height: 50px; vertical-align: middle">
<a href="../../index.html" class="header-archivebox" title="Go to Main Index...">
<img src="../../static/archive.png" alt="Archive Icon">
ArchiveBox
@ -290,10 +295,9 @@
{{title|safe}}
&nbsp;&nbsp;
<a href="#" class="header-toggle">▾</a>
<br/>
<small>
<a href="{{url}}" class="header-url" title="{{url}}">
{{url_str}}
<code>{{url}}</code>
</a>
</small>
</div>
@ -302,27 +306,25 @@
<div class="header-bottom container-fluid">
<div class="row header-bottom-info">
<div class="col-lg-4">
<div title="Date bookmarked or imported" class="info-chunk">
<div title="Date bookmarked or imported" class="info-chunk" title="UTC Timezone {{timestamp}}">
<h5>Added</h5>
{{bookmarked_date}}
</div>
<div title="Date first archived" class="info-chunk">
<div title="Date first archived" class="info-chunk" title="UTC Timezone">
<h5>First Archived</h5>
{{oldest_archive_date}}
</div>
<div title="Date last checked" class="info-chunk">
<div title="Date last checked" class="info-chunk" title="UTC Timezone">
<h5>Last Checked</h5>
{{updated_date}}
</div>
</div>
<div class="col-lg-4">
<div class="info-chunk">
<h5>Type</h5>
<div class="badge badge-default">{{extension}}</div>
</div>
<div class="info-chunk">
<h5>Tags</h5>
<div class="badge badge-warning">{{tags}}</div>
<div class="info-chunk" style="max-width: 280px">
<h5>Tags <small title="Auto-guessed content type">({{extension}})</small></h5>
{% for tag in tags_str|split:',' %}
<div class="badge badge-info" style="word-break: break-all;">{{tag}}</div>
{% endfor %}
</div>
<div class="info-chunk">
<h5>Status</h5>
@ -330,11 +332,11 @@
</div>
<div class="info-chunk">
<h5>Saved</h5>
✅ {{num_outputs}}
✅&nbsp; {{num_outputs}}
</div>
<div class="info-chunk">
<h5>Errors</h5>
❌ {{num_failures}}
❌&nbsp; {{num_failures}}
</div>
<div class="info-chunk">
<h5>Size</h5>
@ -343,7 +345,7 @@
</div>
<div class="col-lg-4">
<div class="info-chunk">
<h5>🗃 Snapshot ID: <a href="/admin/core/snapshot/{{snapshot_id}}/change/"><code style="color: rgba(255,255,255,0.6); font-weight: 200; font-size: 12px; background-color: #1a1a1a"><b>[{{timestamp}}]</b> <small>{{snapshot_id|truncatechars:24}}</small></code></a></h5>
<h5>🗃&nbsp; Snapshot: <a href="/admin/core/snapshot/{{snapshot_id}}/change/"><code style="color: rgba(255,255,255,0.6); font-weight: 200; font-size: 12px; background-color: #1a1a1a"><b>[{{timestamp}}]</b> <small>{{snapshot_id|truncatechars:24}}</small></code></a></h5>
<a href="index.json" title="JSON summary of archived link.">JSON</a> |
<a href="warc/" title="Any WARC archives for the page">WARC</a> |
<a href="media/" title="Audio, Video, and Subtitle files.">Media</a> |
@ -357,7 +359,7 @@
<div class="row header-bottom-frames">
<div class="col-lg-2">
<div class="card selected-card">
<iframe class="card-img-top" src="{{singlefile_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe>
<iframe class="card-img-top" src="{{singlefile_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body">
<a href="{{singlefile_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./singlefile.html</code></p>
@ -368,7 +370,7 @@
</div>
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top pdf-frame" src="{{pdf_path}}" scrolling="no"></iframe>
<iframe class="card-img-top pdf-frame" src="{{pdf_path}}#toolbar=0" scrolling="no" loading="lazy"></iframe>
<div class="card-body">
<a href="{{pdf_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./output.pdf</code></p>
@ -390,7 +392,7 @@
</div>
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top" src="{{archive_url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe>
<iframe class="card-img-top" src="{{archive_url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body">
<a href="{{archive_url}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./{{domain}}</code></p>
@ -402,7 +404,7 @@
{% if SAVE_ARCHIVE_DOT_ORG %}
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top" src="{{archive_org_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe>
<iframe class="card-img-top" src="{{archive_org_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body">
<a href="{{archive_org_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>🌐 web.archive.org/web/...</code></p>
@ -414,7 +416,7 @@
{% endif %}
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top" src="{{url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe>
<iframe class="card-img-top" src="{{url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body">
<a href="{{url}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>🌐 {{domain}}</code></p>
@ -425,7 +427,7 @@
</div>
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top" src="{{headers_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe>
<iframe class="card-img-top" src="{{headers_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body">
<a href="{{headers_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./headers.json</code></p>
@ -436,7 +438,7 @@
</div>
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top" src="{{dom_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe>
<iframe class="card-img-top" src="{{dom_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body">
<a href="{{dom_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./output.html</code></p>
@ -447,7 +449,7 @@
</div>
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top" src="{{readability_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe>
<iframe class="card-img-top" src="{{readability_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body">
<a href="{{readability_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./readability/content.html</code></p>
@ -459,7 +461,7 @@
<br/>
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top" src="{{mercury_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe>
<iframe class="card-img-top" src="{{mercury_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body">
<a href="{{mercury_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./mercury/content.html</code></p>
@ -470,7 +472,7 @@
</div>
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top" src="{{media_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe>
<iframe class="card-img-top" src="{{media_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body">
<a href="{{media_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./media/*.mp4</code></p>
@ -481,7 +483,7 @@
</div>
<div class="col-lg-2">
<div class="card">
<iframe class="card-img-top" src="{{git_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no"></iframe>
<iframe class="card-img-top" src="{{git_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
<div class="card-body">
<a href="{{git_path}}" title="Open in new tab..." target="_blank" rel="noopener">
<p class="card-text"><code>./git/*.git</code></p>

60
archivebox/templates/static/admin.css
View file

@ -1,3 +1,9 @@
* {
-webkit-box-sizing: border-box;
-moz-box-sizing: border-box;
box-sizing: border-box;
}
#logo {
height: 30px;
vertical-align: -6px;
@ -36,6 +42,23 @@ div.breadcrumbs {
padding: 6px 15px;
}
#toolbar #searchbar {
height: 25px;
}
#snapshot-view-mode {
float: right;
margin-bottom: -40px;
display: inline-block;
margin-top: 3px;
margin-right: 10px;
font-size: 14px;
opacity: 0.8;
}
#snapshot-view-mode a {
color: #ccc;
}
body.model-snapshot.change-list div.breadcrumbs,
body.model-snapshot.change-list #content .object-tools {
display: none;
@ -92,6 +115,14 @@ body.model-snapshot.change-list #content .object-tools {
background: none;
margin-right: 0px;
width: auto;
max-height: 40px;
overflow: hidden;
display: block;
}
@media (max-width: 1000px) {
#content #changelist .actions {
max-height: 200px;
}
}
#content #changelist .actions .button {
@ -116,20 +147,45 @@ body.model-snapshot.change-list #content .object-tools {
background-color:lightseagreen;
color: #333;
}
#content #changelist .actions .button[name=resnapshot_snapshot] {
background-color: #9ee54b;
color: #333;
}
#content #changelist .actions .button[name=overwrite_snapshots] {
background-color: #ffaa31;
color: #333;
margin-left: 10px;
}
#content #changelist .actions .button[name=delete_snapshots] {
background-color: #f91f74;
color: rgb(255 248 252 / 64%);
}
#content #changelist .actions .button[name=add_tags] {
}
#content #changelist .actions .button[name=remove_tags] {
margin-right: 25px;
}
#content #changelist .actions .select2-selection {
max-height: 25px;
}
#content #changelist .actions .select2-container--admin-autocomplete.select2-container {
width: auto !important;
min-width: 90px;
}
#content #changelist .actions .select2-selection__rendered .select2-selection__choice {
margin-top: 3px;
}
#content #changelist-filter h2 {
border-radius: 4px 4px 0px 0px;
}
#changelist .paginator {
border-top: 0px;
border-bottom: 0px;
}
@media (min-width: 767px) {
#content #changelist-filter {
top: 35px;
@ -157,7 +213,7 @@ body.model-snapshot.change-list #content .object-tools {
#content a img.favicon {
height: 20px;
width: 20px;
max-width: 28px;
vertical-align: -5px;
padding-right: 6px;
}
@ -177,7 +233,7 @@ body.model-snapshot.change-list #content .object-tools {
#content th.field-added, #content td.field-updated {
word-break: break-word;
min-width: 128px;
min-width: 135px;
white-space: normal;
}

10
archivebox/util.py
View file

@ -11,7 +11,7 @@ from functools import wraps
from hashlib import sha256
from urllib.parse import urlparse, quote, unquote
from html import escape, unescape
from datetime import datetime
from datetime import datetime, timezone
from dateparser import parse as dateparser
from requests.exceptions import RequestException, ReadTimeout
@ -51,7 +51,7 @@ htmlencode = lambda s: s and escape(s, quote=True)
htmldecode = lambda s: s and unescape(s)
short_ts = lambda ts: str(parse_date(ts).timestamp()).split('.')[0]
ts_to_date = lambda ts: ts and parse_date(ts).strftime('%Y-%m-%d %H:%M')
ts_to_date_str = lambda ts: ts and parse_date(ts).strftime('%Y-%m-%d %H:%M')
ts_to_iso = lambda ts: ts and parse_date(ts).isoformat()
@ -144,13 +144,17 @@ def parse_date(date: Any) -> Optional[datetime]:
return None
if isinstance(date, datetime):
if date.tzinfo is None:
return date.replace(tzinfo=timezone.utc)
assert date.tzinfo.utcoffset(datetime.now()).seconds == 0, 'Refusing to load a non-UTC date!'
return date
if isinstance(date, (float, int)):
date = str(date)
if isinstance(date, str):
return dateparser(date)
return dateparser(date, settings={'TIMEZONE': 'UTC'}).replace(tzinfo=timezone.utc)
raise ValueError('Tried to parse invalid date! {}'.format(date))