Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-10 06:34:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

ignore lib dirs

Browse source
This commit is contained in:
Nick Sweeting 2024-09-17 00:47:55 -07:00
parent 8d69469887
commit 25db6826ec
No known key found for this signature in database
4 changed files with 79 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -26,6 +26,7 @@ build/
dist/
# Data folders
lib/
tmp/
data/
data*/

25
archivebox/builtin_plugins/singlefile/apps.py
View file

@ -14,6 +14,7 @@ from plugantic.base_binary import BaseBinary, env
from plugantic.base_extractor import BaseExtractor
from plugantic.base_queue import BaseQueue
from plugantic.base_hook import BaseHook
from plugantic.ansible_utils import run_playbook
# Depends on Other Plugins:
from builtin_plugins.npm.apps import npm
@ -59,12 +60,7 @@ DEFAULT_GLOBAL_CONFIG = {
SINGLEFILE_CONFIG = SinglefileConfigs(**DEFAULT_GLOBAL_CONFIG)
min_version: str = "1.1.54"
max_version: str = "2.0.0"
def get_singlefile_abspath() -> Optional[Path]:
return
INSTALL_BIN = './install_singlefile.yml'
class SinglefileBinary(BaseBinary):
@ -81,6 +77,23 @@ class SinglefileBinary(BaseBinary):
# },
}
def install(self, *args, quiet=False) -> 'SinglefileBinary':
install_playbook = self.plugin_dir / 'install_singlefile.yml'
singlefile_bin = run_playbook(install_playbook, data_dir=settings.CONFIG.OUTPUT_DIR, quiet=quiet).BINARIES.singlefile
return self.__class__.model_validate(
{
**self.model_dump(),
"loaded_abspath": singlefile_bin.abspath,
"loaded_version": singlefile_bin.version,
"loaded_binprovider": env,
"binproviders_supported": self.binproviders_supported,
}
)
SINGLEFILE_BINARY = SinglefileBinary()
PLUGIN_BINARIES = [SINGLEFILE_BINARY]

76
archivebox/playbooks/runner.py
View file

@ -1,76 +0,0 @@
import os
from ansible_runner import Runner, RunnerConfig
from benedict import benedict
from rich.pretty import pprint
GLOBAL_CACHE = {}
IGNORED_VARS = ('OUTPUT', 'STDOUT', 'STDERR', 'RC', 'CMD')
# IGNORED_VARS = ()
os.environ['ANSIBLE_INVENTORY_UNPARSED_WARNING'] = 'False'
os.environ['ANSIBLE_LOCALHOST_WARNING'] = 'False'
def run_playbook(name, extravars=None, getvars=IGNORED_VARS):
_discarded = [GLOBAL_CACHE.pop(key) for key in IGNORED_VARS if key in GLOBAL_CACHE]
rc = RunnerConfig(
private_data_dir=".",
playbook=f"/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/archivebox/playbooks/{name}",
rotate_artifacts=50000,
host_pattern="localhost",
extravars={
**(extravars or {}),
"DATA_DIR": "/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4",
},
# quiet=True,
)
rc.prepare()
r = Runner(config=rc)
r.set_fact_cache('localhost', GLOBAL_CACHE)
r.run()
last_run_facts = r.get_fact_cache('localhost')
GLOBAL_CACHE.update(filtered_facts(last_run_facts))
return benedict({
key: val
for key, val in last_run_facts.items()
if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup'))
})
def filtered_facts(facts):
return benedict({
key: val
for key, val in facts.items()
if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup', *IGNORED_VARS))
})
def print_globals():
pprint(filtered_facts(GLOBAL_CACHE), expand_all=True)
# for each_host_event in r.events:
# print(each_host_event['event'])
# print("Final status:")
# print(r.stats)
ALL_VARS = run_playbook('install_all.yml')
# pprint(ALL_VARS)
print_globals()
# YTDLP_BIN = run_playbook('bin_path.yml', getvars=('OUTPUT', 'STDOUT', 'STDERR', 'RC', 'CMD', 'ytdlp_bin_abs', 'hostvars'))
# pprint(YTDLP_BIN.YTDLP_BIN_ABS)
# print_globals()
# YTDLP_VERSION = run_playbook('version.yml') #, {'YTDLP_BIN': YTDLP_BIN.OUTPUT})
# pprint(YTDLP_VERSION.YTDLP_VERSION)
# print_globals()
# YTDLP_OUTPUT = run_playbook('extract.yml', {'url': 'https://www.youtube.com/watch?v=cK4REjqGc9w&t=27s'})
# pprint(YTDLP_OUTPUT)
# print()
# print()
# print_globals()

59
archivebox/plugantic/ansible_utils.py Normal file
View file

@ -0,0 +1,59 @@
import os
from pathlib import Path
from benedict import benedict
from rich.pretty import pprint
from ansible_runner import Runner, RunnerConfig
GLOBAL_CACHE = {}
def run_playbook(playbook_path, data_dir, quiet=False, **kwargs):
ANSIBLE_TMP_DIR = str(Path(data_dir) / "tmp" / "ansible")
os.environ['ANSIBLE_INVENTORY_UNPARSED_WARNING'] = 'False'
os.environ['ANSIBLE_LOCALHOST_WARNING'] = 'False'
os.environ["ANSIBLE_HOME"] = ANSIBLE_TMP_DIR
# os.environ["ANSIBLE_COLLECTIONS_PATH"] = str(Path(data_dir).parent / 'archivebox')
os.environ["ANSIBLE_ROLES_PATH"] = (
'/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/archivebox/builtin_plugins/ansible/roles'
)
rc = RunnerConfig(
private_data_dir=ANSIBLE_TMP_DIR,
playbook=str(playbook_path),
rotate_artifacts=50000,
host_pattern="localhost",
extravars={
"DATA_DIR": str(data_dir),
**kwargs,
},
quiet=quiet,
)
rc.prepare()
r = Runner(config=rc)
r.set_fact_cache('localhost', GLOBAL_CACHE)
r.run()
last_run_facts = r.get_fact_cache('localhost')
GLOBAL_CACHE.update(filtered_facts(last_run_facts))
return benedict({
key: val
for key, val in last_run_facts.items()
if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup'))
})
def filtered_facts(facts):
return benedict({
key: val
for key, val in facts.items()
if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup'))
})
def print_globals():
pprint(filtered_facts(GLOBAL_CACHE), expand_all=True)
# YTDLP_OUTPUT = run_playbook('extract.yml', {'url': 'https://www.youtube.com/watch?v=cK4REjqGc9w&t=27s'})
# pprint(YTDLP_OUTPUT)