mirror of
https://github.com/ArchiveBox/ArchiveBox
synced 2024-11-26 06:00:22 +00:00
tweak console output format
This commit is contained in:
Nick Sweeting
parent
262fa0e1bb
commit
64e6eb5f7b
3 changed files with 41 additions and 35 deletions
22
archive
22
archive
|
|
@ -54,18 +54,20 @@ def merge_links(archive_path=HTML_FOLDER, import_path=None):
|
|||
all_links = validate_links(existing_links + all_links)
|
||||
|
||||
num_new_links = len(all_links) - len(existing_links)
|
||||
if import_path:
|
||||
print('[+] [{}] Adding {} new links from {} to index'.format(
|
||||
if num_new_links:
|
||||
print('[{green}+{reset}] [{}] Adding {} new links from {} to {}/index.json'.format(
|
||||
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
num_new_links,
|
||||
import_path,
|
||||
archive_path,
|
||||
**ANSI,
|
||||
))
|
||||
else:
|
||||
print('[*] [{}] Running on existing index with {}{}{} links.'.format(
|
||||
print('[*] [{}] No new links added to {}/index.json{}'.format(
|
||||
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
ANSI['green'],
|
||||
len(all_links),
|
||||
ANSI['reset'],
|
||||
archive_path,
|
||||
' from {}'.format(import_path) if import_path else '',
|
||||
**ANSI,
|
||||
))
|
||||
|
||||
return all_links
|
||||
|
|
@ -76,14 +78,15 @@ def update_archive(archive_path, links, source=None, resume=None, append=True):
|
|||
start_ts = datetime.now().timestamp()
|
||||
|
||||
if resume:
|
||||
print('{green}[▶] [{}] Resuming archive update from {}...{reset}'.format(
|
||||
print('{green}[▶] [{}] Resuming archive downloading from {}...{reset}'.format(
|
||||
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
resume,
|
||||
**ANSI,
|
||||
))
|
||||
else:
|
||||
print('{green}[▶] [{}] Running full archive update...{reset}'.format(
|
||||
print('{green}[▶] [{}] Updating files for {} links in archive...{reset}'.format(
|
||||
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
len(links),
|
||||
**ANSI,
|
||||
))
|
||||
|
||||
|
|
@ -98,9 +101,10 @@ def update_archive(archive_path, links, source=None, resume=None, append=True):
|
|||
else:
|
||||
duration = '{0:.2f} sec'.format(seconds, 2)
|
||||
|
||||
print('{}[√] [{}] Archive update complete ({}){}'.format(
|
||||
print('{}[√] [{}] Update of {} links complete ({}){}'.format(
|
||||
ANSI['green'],
|
||||
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
len(links),
|
||||
duration,
|
||||
ANSI['reset'],
|
||||
))
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ def archive_links(archive_path, links, source=None, resume=None):
|
|||
total=len(links),
|
||||
))
|
||||
print(' Continue where you left off by running:')
|
||||
print(' {} {} {}'.format(
|
||||
print(' {} {} {}'.format(
|
||||
sys.argv[0],
|
||||
source,
|
||||
link['timestamp'],
|
||||
|
|
@ -111,14 +111,15 @@ def archive_link(link_dir, link, overwrite=False):
|
|||
return link
|
||||
|
||||
def log_link_archive(link_dir, link, update_existing):
|
||||
print('[{symbol_color}{symbol}{reset}] [{timestamp}] "{title}": {blue}{base_url}{reset}'.format(
|
||||
print('[{symbol_color}{symbol}{reset}] [{now}] "{title}"\n {blue}{url}{reset}'.format(
|
||||
symbol='*' if update_existing else '+',
|
||||
symbol_color=ANSI['black' if update_existing else 'green'],
|
||||
now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
**link,
|
||||
**ANSI,
|
||||
))
|
||||
|
||||
print(' > {} ({})'.format(link_dir, 'updating' if update_existing else 'creating'))
|
||||
print(' > {}{}'.format(link_dir, '' if update_existing else ' (new)'))
|
||||
if link['type']:
|
||||
print(' i {}'.format(link['type']))
|
||||
|
||||
|
|
@ -196,15 +197,16 @@ def fetch_wget(link_dir, link, requisites=FETCH_WGET_REQUISITES, timeout=TIMEOUT
|
|||
try:
|
||||
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout + 1) # index.html
|
||||
end()
|
||||
output = html_appended_url(link)
|
||||
if result.returncode > 0:
|
||||
print(' got wget response code {}:'.format(result.returncode))
|
||||
print('\n'.join(' ' + line for line in (result.stderr or result.stdout).decode().rsplit('\n', 10)[-10:] if line.strip()))
|
||||
# raise Exception('Failed to wget download')
|
||||
output = wget_output_path(link, look_in=domain_dir)
|
||||
if result.returncode > 0 and result.returncode != 8:
|
||||
print(' got wget response code {}:'.format(result.returncode))
|
||||
print('\n'.join(' ' + line for line in (result.stderr or result.stdout).decode().rsplit('\n', 10)[-10:] if line.strip()))
|
||||
if result.returncode == 4:
|
||||
raise Exception('Failed to wget download')
|
||||
except Exception as e:
|
||||
end()
|
||||
print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
output = e
|
||||
|
||||
return {
|
||||
|
|
@ -239,8 +241,8 @@ def fetch_pdf(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DI
|
|||
output = 'output.pdf'
|
||||
except Exception as e:
|
||||
end()
|
||||
print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
output = e
|
||||
|
||||
return {
|
||||
|
|
@ -276,8 +278,8 @@ def fetch_screenshot(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_
|
|||
output = 'screenshot.png'
|
||||
except Exception as e:
|
||||
end()
|
||||
print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
output = e
|
||||
|
||||
return {
|
||||
|
|
@ -323,8 +325,8 @@ def archive_dot_org(link_dir, link, timeout=TIMEOUT):
|
|||
raise Exception('Failed to find "Content-Location" URL header in Archive.org response.')
|
||||
except Exception as e:
|
||||
end()
|
||||
print(' Visit url to see output:', ' '.join(CMD))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
print(' Visit url to see output:', ' '.join(CMD))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
output = e
|
||||
|
||||
if success:
|
||||
|
|
@ -357,8 +359,8 @@ def fetch_favicon(link_dir, link, timeout=TIMEOUT):
|
|||
except Exception as e:
|
||||
fout.close()
|
||||
end()
|
||||
print(' Run to see full output:', ' '.join(CMD))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
print(' Run to see full output:', ' '.join(CMD))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
output = e
|
||||
|
||||
return {
|
||||
|
|
@ -393,8 +395,8 @@ def fetch_favicon(link_dir, link, timeout=TIMEOUT):
|
|||
# return 'audio.mp3'
|
||||
# except Exception as e:
|
||||
# end()
|
||||
# print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
|
||||
# print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
# print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
|
||||
# print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
# raise
|
||||
# else:
|
||||
# print(' √ Skipping audio download')
|
||||
|
|
@ -426,8 +428,8 @@ def fetch_favicon(link_dir, link, timeout=TIMEOUT):
|
|||
# return 'video.mp4'
|
||||
# except Exception as e:
|
||||
# end()
|
||||
# print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
|
||||
# print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
# print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
|
||||
# print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
# raise
|
||||
# else:
|
||||
# print(' √ Skipping video download')
|
||||
|
|
|
|||
8
index.py
8
index.py
|
|
@ -33,11 +33,11 @@ def write_links_index(out_dir, links):
|
|||
write_json_links_index(out_dir, links)
|
||||
write_html_links_index(out_dir, links)
|
||||
|
||||
print('[√] [{}] Archive Main Index now up-to-date: {}/index.html'.format(
|
||||
print('{green}[√] [{}] Updated main index files:{reset}'.format(
|
||||
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
out_dir,
|
||||
|
||||
))
|
||||
**ANSI))
|
||||
print(' > {}/index.json'.format(out_dir))
|
||||
print(' > {}/index.html'.format(out_dir))
|
||||
|
||||
def write_json_links_index(out_dir, links):
|
||||
"""write the json link index to a given path"""
|
||||
|
|
|
|||
Loading…
Reference in a new issue