From 25db6826ec45f8cc7626a269b4a98ec7e1c6ba3d Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 17 Sep 2024 00:47:55 -0700 Subject: [PATCH] ignore lib dirs --- .gitignore | 1 + archivebox/builtin_plugins/singlefile/apps.py | 25 ++++-- archivebox/playbooks/runner.py | 76 ------------------- archivebox/plugantic/ansible_utils.py | 59 ++++++++++++++ 4 files changed, 79 insertions(+), 82 deletions(-) delete mode 100644 archivebox/playbooks/runner.py create mode 100644 archivebox/plugantic/ansible_utils.py diff --git a/.gitignore b/.gitignore index 64fde262..758458c7 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ build/ dist/ # Data folders +lib/ tmp/ data/ data*/ diff --git a/archivebox/builtin_plugins/singlefile/apps.py b/archivebox/builtin_plugins/singlefile/apps.py index e226cc66..eeb41681 100644 --- a/archivebox/builtin_plugins/singlefile/apps.py +++ b/archivebox/builtin_plugins/singlefile/apps.py @@ -14,6 +14,7 @@ from plugantic.base_binary import BaseBinary, env from plugantic.base_extractor import BaseExtractor from plugantic.base_queue import BaseQueue from plugantic.base_hook import BaseHook +from plugantic.ansible_utils import run_playbook # Depends on Other Plugins: from builtin_plugins.npm.apps import npm @@ -59,12 +60,7 @@ DEFAULT_GLOBAL_CONFIG = { SINGLEFILE_CONFIG = SinglefileConfigs(**DEFAULT_GLOBAL_CONFIG) - -min_version: str = "1.1.54" -max_version: str = "2.0.0" - -def get_singlefile_abspath() -> Optional[Path]: - return +INSTALL_BIN = './install_singlefile.yml' class SinglefileBinary(BaseBinary): @@ -80,6 +76,23 @@ class SinglefileBinary(BaseBinary): # 'packages': lambda: f'single-file-cli@>={min_version} <{max_version}', # }, } + + def install(self, *args, quiet=False) -> 'SinglefileBinary': + + install_playbook = self.plugin_dir / 'install_singlefile.yml' + + singlefile_bin = run_playbook(install_playbook, data_dir=settings.CONFIG.OUTPUT_DIR, quiet=quiet).BINARIES.singlefile + + return self.__class__.model_validate( + { + **self.model_dump(), + "loaded_abspath": singlefile_bin.abspath, + "loaded_version": singlefile_bin.version, + "loaded_binprovider": env, + "binproviders_supported": self.binproviders_supported, + } + ) + SINGLEFILE_BINARY = SinglefileBinary() diff --git a/archivebox/playbooks/runner.py b/archivebox/playbooks/runner.py deleted file mode 100644 index f4d21c6c..00000000 --- a/archivebox/playbooks/runner.py +++ /dev/null @@ -1,76 +0,0 @@ -import os -from ansible_runner import Runner, RunnerConfig -from benedict import benedict -from rich.pretty import pprint - - -GLOBAL_CACHE = {} - -IGNORED_VARS = ('OUTPUT', 'STDOUT', 'STDERR', 'RC', 'CMD') -# IGNORED_VARS = () - -os.environ['ANSIBLE_INVENTORY_UNPARSED_WARNING'] = 'False' -os.environ['ANSIBLE_LOCALHOST_WARNING'] = 'False' - -def run_playbook(name, extravars=None, getvars=IGNORED_VARS): - _discarded = [GLOBAL_CACHE.pop(key) for key in IGNORED_VARS if key in GLOBAL_CACHE] - rc = RunnerConfig( - private_data_dir=".", - playbook=f"/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/archivebox/playbooks/{name}", - rotate_artifacts=50000, - host_pattern="localhost", - extravars={ - **(extravars or {}), - "DATA_DIR": "/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4", - }, - # quiet=True, - ) - rc.prepare() - r = Runner(config=rc) - r.set_fact_cache('localhost', GLOBAL_CACHE) - r.run() - last_run_facts = r.get_fact_cache('localhost') - GLOBAL_CACHE.update(filtered_facts(last_run_facts)) - return benedict({ - key: val - for key, val in last_run_facts.items() - if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup')) - }) - -def filtered_facts(facts): - return benedict({ - key: val - for key, val in facts.items() - if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup', *IGNORED_VARS)) - }) - -def print_globals(): - pprint(filtered_facts(GLOBAL_CACHE), expand_all=True) - -# for each_host_event in r.events: -# print(each_host_event['event']) - -# print("Final status:") -# print(r.stats) - -ALL_VARS = run_playbook('install_all.yml') -# pprint(ALL_VARS) -print_globals() - - -# YTDLP_BIN = run_playbook('bin_path.yml', getvars=('OUTPUT', 'STDOUT', 'STDERR', 'RC', 'CMD', 'ytdlp_bin_abs', 'hostvars')) -# pprint(YTDLP_BIN.YTDLP_BIN_ABS) -# print_globals() - - -# YTDLP_VERSION = run_playbook('version.yml') #, {'YTDLP_BIN': YTDLP_BIN.OUTPUT}) -# pprint(YTDLP_VERSION.YTDLP_VERSION) -# print_globals() - - -# YTDLP_OUTPUT = run_playbook('extract.yml', {'url': 'https://www.youtube.com/watch?v=cK4REjqGc9w&t=27s'}) -# pprint(YTDLP_OUTPUT) - -# print() -# print() -# print_globals() diff --git a/archivebox/plugantic/ansible_utils.py b/archivebox/plugantic/ansible_utils.py new file mode 100644 index 00000000..7288c971 --- /dev/null +++ b/archivebox/plugantic/ansible_utils.py @@ -0,0 +1,59 @@ +import os + +from pathlib import Path + +from benedict import benedict +from rich.pretty import pprint + +from ansible_runner import Runner, RunnerConfig + +GLOBAL_CACHE = {} + + +def run_playbook(playbook_path, data_dir, quiet=False, **kwargs): + ANSIBLE_TMP_DIR = str(Path(data_dir) / "tmp" / "ansible") + os.environ['ANSIBLE_INVENTORY_UNPARSED_WARNING'] = 'False' + os.environ['ANSIBLE_LOCALHOST_WARNING'] = 'False' + os.environ["ANSIBLE_HOME"] = ANSIBLE_TMP_DIR + # os.environ["ANSIBLE_COLLECTIONS_PATH"] = str(Path(data_dir).parent / 'archivebox') + os.environ["ANSIBLE_ROLES_PATH"] = ( + '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/archivebox/builtin_plugins/ansible/roles' + ) + + rc = RunnerConfig( + private_data_dir=ANSIBLE_TMP_DIR, + playbook=str(playbook_path), + rotate_artifacts=50000, + host_pattern="localhost", + extravars={ + "DATA_DIR": str(data_dir), + **kwargs, + }, + quiet=quiet, + ) + rc.prepare() + r = Runner(config=rc) + r.set_fact_cache('localhost', GLOBAL_CACHE) + r.run() + last_run_facts = r.get_fact_cache('localhost') + GLOBAL_CACHE.update(filtered_facts(last_run_facts)) + return benedict({ + key: val + for key, val in last_run_facts.items() + if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup')) + }) + +def filtered_facts(facts): + return benedict({ + key: val + for key, val in facts.items() + if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup')) + }) + +def print_globals(): + pprint(filtered_facts(GLOBAL_CACHE), expand_all=True) + + + +# YTDLP_OUTPUT = run_playbook('extract.yml', {'url': 'https://www.youtube.com/watch?v=cK4REjqGc9w&t=27s'}) +# pprint(YTDLP_OUTPUT)