diff --git a/archivebox/builtin_plugins/ansible/roles/install_packages/tasks/main.yml b/archivebox/builtin_plugins/ansible/roles/install_packages/tasks/main.yml new file mode 100755 index 00000000..409507d7 --- /dev/null +++ b/archivebox/builtin_plugins/ansible/roles/install_packages/tasks/main.yml @@ -0,0 +1,58 @@ +#!/usr/bin/env ansible-playbook +--- +# - name: "Install using apt/brew/pkg/yum/etc." +# hosts: localhost +# gather_facts: no +# vars: +# DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4' +# LIB_DIR: '{{DATA_DIR}}/lib' +# LIB_DIR_BIN: '{{LIB_DIR}}/bin' +# state: 'present' +# install_packages: {} +# # bash: +# # packages: ['bash'] +# PACKAGE_BINPROVIDERS: {} +# PACKAGE_BINARIES: {} +# BINPROVIDERS: {} +# tasks: +# - package: update_cache=yes + # when: ansible_facts['os_family'] == "Debian" + +- name: Make sure lib folders exist + file: + path: '{{LIB_DIR_BIN}}' + state: directory + recurse: true + when: BINPROVIDERS.ansible_package is not defined + +- name: Get ansible binary abspath + command: 'which ansible' + register: ANSIBLE_INSTALLER_ABSPATH_FULL + changed_when: false + when: BINPROVIDERS.ansible_package is not defined + +################################################################################### +- name: "Install system packages: {{all_packages}}" + ansible.builtin.package: + name: "{{item}}" + state: "{{state}}" + loop: "{{all_packages}}" + +################################################################################### + +- set_fact: + PACKAGE_BINPROVIDERS: + ansible_package: + installer_abspath: "{{ANSIBLE_INSTALLER_ABSPATH_FULL.stdout}}" + installer_version: "{{ansible_version.full}}" + PATH: "/opt/homebrew/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + when: BINPROVIDERS.ansible_package is not defined + +- set_fact: + BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(PACKAGE_BINPROVIDERS) }}" + cacheable: true + when: BINPROVIDERS.ansible_package is not defined + +- debug: + msg: "{{ {'BINPROVIDERS': BINPROVIDERS} }}" + diff --git a/archivebox/builtin_plugins/ansible/roles/install_packages/vars/main.yml b/archivebox/builtin_plugins/ansible/roles/install_packages/vars/main.yml new file mode 100644 index 00000000..0a21d935 --- /dev/null +++ b/archivebox/builtin_plugins/ansible/roles/install_packages/vars/main.yml @@ -0,0 +1,5 @@ +DATA_DIR: '{{playbook_dir}}' +LIB_DIR: '{{DATA_DIR}}/lib' +LIB_DIR_BIN: '{{LIB_DIR}}/bin' +state: present +all_packages: [] diff --git a/archivebox/builtin_plugins/ansible/roles/load_binary/meta/argument_specs.yml b/archivebox/builtin_plugins/ansible/roles/load_binary/meta/argument_specs.yml new file mode 100644 index 00000000..7f95fa87 --- /dev/null +++ b/archivebox/builtin_plugins/ansible/roles/load_binary/meta/argument_specs.yml @@ -0,0 +1,31 @@ +--- +argument_specs: + main: + short_description: Load a specified binary from the environment/PATH into BINARIES fact. + + options: + + name: + type: "str" + required: true + description: "A string key for the binary" + + bin_name: + type: "str" + required: false + description: "The basename of the binary file (optional, defaults to name)" + + abspath: + type: "str" + required: false + description: "An absolute path to the binary (overrides any auto-detected one)" + + version_cmd: + type: "str" + required: false + description: "The command to run to get the binary's version (optional, defaults to $ --version)" + + PATH: + type: "str" + required: false + description: "The PATH to search for the binary (optional, defaults to environment $PATH)" diff --git a/archivebox/builtin_plugins/ansible/roles/load_binary/tasks/main.yml b/archivebox/builtin_plugins/ansible/roles/load_binary/tasks/main.yml new file mode 100755 index 00000000..42f0c5d7 --- /dev/null +++ b/archivebox/builtin_plugins/ansible/roles/load_binary/tasks/main.yml @@ -0,0 +1,73 @@ + +--- +- name: Make sure ./data/lib/bin folder exists + file: + path: '{{LIB_DIR_BIN}}' + state: directory + recurse: true + +################################################################################### + +- set_fact: + PATH: "{{PATH or DEFAULT_PATH}}" + bin_name: "{{bin_name or name}}" + BINARY_ABSPATH: null + BINARY_VERSION: null + +- name: 'Get installed binary abspath: {{name}}' + command: 'env PATH="{{PATH}}:$PATH" which {{bin_name}}' + register: BINARY_ABSPATH + changed_when: BINARIES[name].abspath|default('NO VERSION FOUND') not in BINARY_ABSPATH.stdout + +- set_fact: + bin_name: "{{BINARY_ABSPATH.stdout|basename or bin_name}}" +- set_fact: + version_cmd: "{{version_cmd or (bin_name + ' --version')}}" + +- name: 'Get installed binary version: {{name}}' + command: 'env PATH="{{PATH}}:$PATH" {{version_cmd}}' + register: BINARY_VERSION + changed_when: BINARIES[name].version|default('NO VERSION FOUND') not in BINARY_VERSION.stdout + + +- name: 'Updating BINARIES with loaded abspaths & versions: {{name}}' + set_fact: + BINARIES: "{{ BINARIES + | default({}) + | combine({ + name: { + 'name': name, + 'bin_name': bin_name, + 'version_cmd': version_cmd, + 'symlink': LIB_DIR_BIN + '/' + name, + 'abspath': BINARY_ABSPATH.stdout or abspath or None, + 'version': BINARY_VERSION.stdout_lines|first|regex_replace('^.*?([\\d+\\.]+).*$', '\\1') or version or None, + 'version_stdout': BINARY_VERSION.stdout or BINARY_VERSION.stderr, + 'PATH': BINARY_ABSPATH.stdout|dirname or PATH, + }, + }) + }}" + cacheable: true + when: BINARY_ABSPATH.stdout and BINARY_VERSION.stdout + +- name: 'Symlink installed binary into lib bin folder: {{name}}' + file: + src: "{{ BINARY_ABSPATH.stdout }}" + dest: "{{ LIB_DIR_BIN }}/{{ name }}" + state: link + force: true + when: BINARY_ABSPATH.stdout and BINARY_VERSION.stdout + changed_when: False + +- debug: + msg: + - '{{BINARIES}}' + +- name: Unset variables + set_fact: + name: + bin_name: + version_cmd: + PATH: + BINARY_ABSPATH: + BINARY_VERSION: diff --git a/archivebox/builtin_plugins/ansible/roles/load_binary/vars/main.yml b/archivebox/builtin_plugins/ansible/roles/load_binary/vars/main.yml new file mode 100644 index 00000000..c8ba73b0 --- /dev/null +++ b/archivebox/builtin_plugins/ansible/roles/load_binary/vars/main.yml @@ -0,0 +1,11 @@ +DATA_DIR: '{{playbook_dir}}' +LIB_DIR: '{{DATA_DIR}}/lib' +LIB_DIR_BIN: '{{LIB_DIR}}/bin' +DEFAULT_PATH: /bin + +name: +bin_name: +version_cmd: +PATH: +abspath: +version: diff --git a/archivebox/builtin_plugins/ansible/roles/setup_lib_npm/meta/argument_specs.yml b/archivebox/builtin_plugins/ansible/roles/setup_lib_npm/meta/argument_specs.yml new file mode 100644 index 00000000..df0647d0 --- /dev/null +++ b/archivebox/builtin_plugins/ansible/roles/setup_lib_npm/meta/argument_specs.yml @@ -0,0 +1,31 @@ +--- +argument_specs: + main: + short_description: Main entry point for the npm role + + options: + + state: + type: "str" + required: false + default: 'present' + description: + - "The desired state: present | latest" + + npm_packages: + type: "list" + elements: "dict" + required: false + default: [] + description: "A list of dicts with a defined structure and with default a value." + options: + key: + type: "str" + required: true + description: "A string name for the dependency" + + packages: + type: "list" + elements: "str" + required: true + description: "What npm packages to install for the given dependency." diff --git a/archivebox/builtin_plugins/ansible/roles/setup_lib_npm/tasks/main.yml b/archivebox/builtin_plugins/ansible/roles/setup_lib_npm/tasks/main.yml new file mode 100755 index 00000000..b11bd915 --- /dev/null +++ b/archivebox/builtin_plugins/ansible/roles/setup_lib_npm/tasks/main.yml @@ -0,0 +1,59 @@ +--- +- name: Make sure lib folders exist + file: + path: '{{item}}' + state: directory + recurse: true + loop: + - '{{LIB_DIR_NPM_BIN}}' + - '{{LIB_DIR_BIN}}' + +- name: "Install system packages: [node, npm]" + ansible.builtin.package: + name: "node" + state: "present" + +- name: Load NPM and Node binaries + include_role: + name: load_binary + vars: + name: '{{item}}' + loop: + - node + - npm + +- name: Check that installed Node version matches expected version + assert: + that: + - BINARIES.node.version is version(MIN_NODE_VERSION, '>=') + - BINARIES.npm.version is version(MIN_NPM_VERSION, '>=') + quiet: true + +################################################################################### + + +# - name: "Install npm packages: {{install_npm}}" +# community.general.npm: +# name: '{{item}}' +# state: "{{state}}" +# path: '{{LIB_DIR_NPM}}' +# loop: "{{install_npm|dictsort|map(attribute='1')|map(attribute='packages')|flatten}}" + +################################################################################### + + +################################################################################### +- set_fact: + NODE_BINPROVIDERS: + npm: + installer_abspath: "{{BINARIES.npm.abspath}}" + installer_version: "{{BINARIES.npm.version}}" + PATH: "{{LIB_DIR_NPM_BIN}}" + lib_dir_npm: "{{LIB_DIR_NPM}}" + +- set_fact: + BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(NODE_BINPROVIDERS) }}" + cacheable: true + +- debug: + msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" diff --git a/archivebox/builtin_plugins/ansible/roles/setup_lib_npm/vars/main.yml b/archivebox/builtin_plugins/ansible/roles/setup_lib_npm/vars/main.yml new file mode 100644 index 00000000..82fb2ea8 --- /dev/null +++ b/archivebox/builtin_plugins/ansible/roles/setup_lib_npm/vars/main.yml @@ -0,0 +1,7 @@ +DATA_DIR: '{{playbook_dir}}' +LIB_DIR: '{{DATA_DIR}}/lib' +LIB_DIR_BIN: '{{LIB_DIR}}/bin' +LIB_DIR_NPM: '{{LIB_DIR}}/npm' +LIB_DIR_NPM_BIN: '{{LIB_DIR_NPM}}/node_modules/.bin' +MIN_NODE_VERSION: '20.0.0' +MIN_NPM_VERSION: '10.0.0' diff --git a/archivebox/builtin_plugins/ansible/roles/setup_lib_pip/meta/argument_specs.yml b/archivebox/builtin_plugins/ansible/roles/setup_lib_pip/meta/argument_specs.yml new file mode 100644 index 00000000..df0647d0 --- /dev/null +++ b/archivebox/builtin_plugins/ansible/roles/setup_lib_pip/meta/argument_specs.yml @@ -0,0 +1,31 @@ +--- +argument_specs: + main: + short_description: Main entry point for the npm role + + options: + + state: + type: "str" + required: false + default: 'present' + description: + - "The desired state: present | latest" + + npm_packages: + type: "list" + elements: "dict" + required: false + default: [] + description: "A list of dicts with a defined structure and with default a value." + options: + key: + type: "str" + required: true + description: "A string name for the dependency" + + packages: + type: "list" + elements: "str" + required: true + description: "What npm packages to install for the given dependency." diff --git a/archivebox/builtin_plugins/ansible/roles/setup_lib_pip/tasks/main.yml b/archivebox/builtin_plugins/ansible/roles/setup_lib_pip/tasks/main.yml new file mode 100755 index 00000000..8676acbc --- /dev/null +++ b/archivebox/builtin_plugins/ansible/roles/setup_lib_pip/tasks/main.yml @@ -0,0 +1,59 @@ +--- +- name: Make sure lib folders exist + file: + path: '{{item}}' + state: directory + recurse: true + loop: + - '{{LIB_DIR_PIP}}' + - '{{LIB_DIR_BIN}}' + when: BINPROVIDERS.pip is not defined + +- name: Load Python and Pip binaries + import_role: + name: load_binary + vars: + name: '{{item}}' + loop: + - python + - pip + when: BINARIES.python is not defined or BINARIES.pip is not defined + +- assert: + that: + - BINARIES.python.version is version(MIN_PYTHON_VERSION, '>=') + - BINARIES.pip.version is version(MIN_PIP_VERSION, '>=') + quiet: true + when: BINPROVIDERS.pip is not defined + +################################################################################### + + +# - name: "Install pip packages: {{install_pip}}" +# ansible.builtin.pip: +# name: '{{item}}' +# state: "{{state}}" +# virtualenv: '{{LIB_DIR_PIP}}/venv' +# virtualenv_python: "{{BINARIES.python.abspath}}" +# virtualenv_site_packages: yes +# loop: "{{install_pip|dictsort|map(attribute='1')|map(attribute='packages')|flatten}}" + + +################################################################################### +- set_fact: + PIP_BINPROVIDERS: + pip: + installer_abspath: "{{BINARIES.pip.abspath}}" + installer_version: "{{BINARIES.pip.version}}" + PATH: "{{LIB_DIR_PIP_BIN}}" + virtualenv: "{{LIB_DIR_PIP}}/venv" + virtualenv_python: "{{BINARIES.python.abspath}}" + when: BINPROVIDERS.pip is not defined + +- set_fact: + BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(PIP_BINPROVIDERS) }}" + cacheable: true + when: BINPROVIDERS.pip is not defined + +- debug: + msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" diff --git a/archivebox/builtin_plugins/ansible/roles/setup_lib_pip/vars/main.yml b/archivebox/builtin_plugins/ansible/roles/setup_lib_pip/vars/main.yml new file mode 100644 index 00000000..0c8ba99b --- /dev/null +++ b/archivebox/builtin_plugins/ansible/roles/setup_lib_pip/vars/main.yml @@ -0,0 +1,7 @@ +DATA_DIR: '{{playbook_dir}}' +LIB_DIR: '{{DATA_DIR}}/lib' +LIB_DIR_BIN: '{{LIB_DIR}}/bin' +LIB_DIR_PIP: '{{LIB_DIR}}/pip' +LIB_DIR_PIP_BIN: '{{LIB_DIR_PIP}}/venv/bin' +MIN_PYTHON_VERSION: '3.10.0' +MIN_PIP_VERSION: '22.0' diff --git a/archivebox/builtin_plugins/puppeteer/__init__.py b/archivebox/builtin_plugins/puppeteer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/archivebox/builtin_plugins/puppeteer/apps.py b/archivebox/builtin_plugins/puppeteer/apps.py new file mode 100644 index 00000000..b44c9f9b --- /dev/null +++ b/archivebox/builtin_plugins/puppeteer/apps.py @@ -0,0 +1,85 @@ +from typing import List, Optional + +from django.conf import settings + +# Depends on other PyPI/vendor packages: +from pydantic import InstanceOf, Field +from pydantic_pkgr import BinProvider, BinName + +# Depends on other Django apps: +from plugantic.base_plugin import BasePlugin +from plugantic.base_configset import BaseConfigSet, ConfigSectionName +from plugantic.base_binary import BaseBinary, env +# from plugantic.base_extractor import BaseExtractor +# from plugantic.base_queue import BaseQueue +from plugantic.base_hook import BaseHook +from plugantic.ansible_utils import run_playbook + +# Depends on Other Plugins: +from builtin_plugins.npm.apps import npm + + +###################### Config ########################## + + +class PuppeteerDependencyConfigs(BaseConfigSet): + section: ConfigSectionName = 'DEPENDENCY_CONFIG' + + PUPPETEER_BINARY: str = Field(default='wget') + PUPPETEER_ARGS: Optional[List[str]] = Field(default=None) + PUPPETEER_EXTRA_ARGS: List[str] = [] + PUPPETEER_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}'] + +class PuppeteerConfigs(PuppeteerDependencyConfigs): + # section: ConfigSectionName = 'ALL_CONFIGS' + pass + +DEFAULT_GLOBAL_CONFIG = { +} + +PUPPETEER_CONFIG = PuppeteerConfigs(**DEFAULT_GLOBAL_CONFIG) + + +INSTALL_BIN = './install_puppeteer.yml' + + +class ChromeBinary(BaseBinary): + name: BinName = 'chrome' + binproviders_supported: List[InstanceOf[BinProvider]] = [npm, env] + + + def install(self, *args, quiet=False) -> "ChromeBinary": + + install_playbook = self.plugin_dir / 'install_puppeteer.yml' + + chrome_bin = run_playbook(install_playbook, data_dir=settings.CONFIG.OUTPUT_DIR, quiet=quiet).BINARIES.chrome + + return self.__class__.model_validate( + { + **self.model_dump(), + "loaded_abspath": chrome_bin.symlink, + "loaded_version": chrome_bin.version, + "loaded_binprovider": env, + "binproviders_supported": self.binproviders_supported, + } + ) + + +CHROME_BINARY = ChromeBinary() + +PLUGIN_BINARIES = [CHROME_BINARY] + +class PuppeteerPlugin(BasePlugin): + app_label: str ='puppeteer' + verbose_name: str = 'SingleFile' + + hooks: List[InstanceOf[BaseHook]] = [ + PUPPETEER_CONFIG, + CHROME_BINARY, + ] + + + +PLUGIN = PuppeteerPlugin() +PLUGIN.register(settings) +DJANGO_APP = PLUGIN.AppConfig diff --git a/archivebox/builtin_plugins/puppeteer/install_puppeteer.yml b/archivebox/builtin_plugins/puppeteer/install_puppeteer.yml new file mode 100755 index 00000000..0f683eb5 --- /dev/null +++ b/archivebox/builtin_plugins/puppeteer/install_puppeteer.yml @@ -0,0 +1,127 @@ +#!/usr/bin/env ansible-playbook +--- +- name: "Install puppeteer, puppeteer/browsers, and chrome" + hosts: localhost + gather_facts: true + vars: + LIB_DIR: '{{DATA_DIR}}/lib' + LIB_DIR_BIN: '{{LIB_DIR}}/bin' + LIB_DIR_BROWSERS: '{{LIB_DIR}}/browsers' + CHROME_RELEASE_CHANNEL: 'chrome@stable' + CHROME_VERSION_MIN: '128.0.6613.137' + tasks: + - include_role: + name: setup_lib_npm + vars: + MIN_NODE_VERSION: '20.0.0' + MIN_NPM_VERSION: '10.0.0' + + - name: "Install npm packages: [puppeteer, @puppeteer/browsers]" + community.general.npm: + name: '{{item}}' + state: "present" + path: '{{BINPROVIDERS.npm.lib_dir_npm}}' + loop: + - 'puppeteer' + - '@puppeteer/browsers' + + - name: Make sure prerequisite folders exist + file: + path: '{{LIB_DIR_BROWSERS}}' + state: directory + recurse: true + + - name: Load puppeteer binary from installed NPM package + include_role: + name: load_binary + vars: + name: puppeteer + PATH: '{{BINPROVIDERS.npm.PATH}}' + + - name: Load chrome binaries from environment PATH + include_tasks: roles/load_binary/tasks/main.yml + vars: + name: 'chrome' + bin_name: '{{chrome_executable}}' + PATH: '{{ansible_env.PATH}}' + with_items: + - chrome + # - chrome-browser + # - chromium + # - chromium-browser + # - google-chrome + # - google-chrome-browser + # - google-chrome-stable + # - google-chrome-beta + # - google-chrome-canary + # - google-chrome-unstable + # - google-chrome-dev + loop_control: + loop_var: chrome_executable + # break_when: + # - BINARIES.chrome.version|default('') + + # - debug: + # msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" + + + ################################################################################### + - name: 'Install Chrome browser: npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}}' + command: 'npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}} --path {{LIB_DIR_BROWSERS}}' + register: CHROME_VERSION_FULL + environment: + PATH: "{{BINPROVIDERS.npm.PATH}}:{{ ansible_env.PATH }}" + changed_when: CHROME_VERSION_MIN not in CHROME_VERSION_FULL.stdout + when: not BINARIES.chrome.version|default('') + # -> 'chrome@128.0.6613.137 /data/lib/browsers/chrome/linux_arm-128.0.6613.138/chrome-linux-arm64/...' + + ################################################################################### + - name: Parse Chrome version and abspath from npx @puppeteer/browsers install output + set_fact: + CHROME_ABSPATH: "{{ CHROME_VERSION_FULL.stdout_lines|last|split(' ', 1)|last }}" + CHROME_VERSION: "{{ CHROME_VERSION_FULL.stdout_lines|last|split('@', 1)|last|split(' ', 1)|first }}" + when: not BINARIES.chrome.version|default('') + + - name: Create ./bin/chrome symlink to ./browsers/chrome/... binary + copy: + content: | + #!/bin/bash + exec '{{CHROME_ABSPATH|default(BINARIES.chrome.abspath)}}' "$@" + dest: "{{LIB_DIR_BIN}}/chrome" + changed_when: False + + - name: Ensure ./bin/chrome symlink is executable + file: + path: "{{LIB_DIR_BIN}}/chrome" + mode: u+rx,g-rx,o-rwx + state: 'file' + changed_when: False + + ################################################################################### + - set_fact: + PUPPETEER_BINARIES: + chrome: + bin_name: 'chrome' + abspath: "{{CHROME_ABSPATH|default(BINARIES.chrome.abspath) or None}}" + version: "{{CHROME_VERSION|default(BINARIES.chrome.version) or None}}" + symlink: "{{LIB_DIR_BIN}}/chrome" + version_cmd: "chrome --version" + binprovider: 'puppeteer' + PUPPETEER_BINPROVIDERS: + puppeteer: + installer_abspath: "{{BINARIES.puppeteer.abspath}}" + installer_version: "{{BINARIES.puppeteer.version}}" + PATH: "{{LIB_DIR_BIN}}" + lib_dir_browsers: "{{LIB_DIR_BROWSERS}}" + + - name: Check that installed Chrome matches expected version + assert: + that: PUPPETEER_BINARIES.chrome.version is version(CHROME_VERSION_MIN, '>=') + quiet: true + + - set_fact: + BINARIES: "{{ BINARIES | combine(PUPPETEER_BINARIES) }}" + cacheable: true + + - debug: + msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" diff --git a/archivebox/builtin_plugins/puppeteer/roles b/archivebox/builtin_plugins/puppeteer/roles new file mode 120000 index 00000000..e4109d37 --- /dev/null +++ b/archivebox/builtin_plugins/puppeteer/roles @@ -0,0 +1 @@ +../ansible/roles \ No newline at end of file diff --git a/archivebox/builtin_plugins/singlefile/install_singlefile.yml b/archivebox/builtin_plugins/singlefile/install_singlefile.yml new file mode 120000 index 00000000..47d85406 --- /dev/null +++ b/archivebox/builtin_plugins/singlefile/install_singlefile.yml @@ -0,0 +1 @@ +../../playbooks/install_singlefile.yml \ No newline at end of file diff --git a/archivebox/builtin_plugins/singlefile/roles b/archivebox/builtin_plugins/singlefile/roles new file mode 120000 index 00000000..e4109d37 --- /dev/null +++ b/archivebox/builtin_plugins/singlefile/roles @@ -0,0 +1 @@ +../ansible/roles \ No newline at end of file diff --git a/archivebox/builtin_plugins/ytdlp/install_ytdlp.yml b/archivebox/builtin_plugins/ytdlp/install_ytdlp.yml new file mode 100755 index 00000000..42cbf48b --- /dev/null +++ b/archivebox/builtin_plugins/ytdlp/install_ytdlp.yml @@ -0,0 +1,42 @@ +#!/usr/bin/env ansible-playbook +--- +- name: "Install YT-DLP" + hosts: localhost + gather_facts: no + vars: + YTDLP_VERSION_MIN: '2024.8.6' + tasks: + - include_role: + name: setup_lib_pip + vars: + MIN_PYTHON_VERSION: '3.10.0' + MIN_PIP_VERSION: '22.0' + + - name: "Install pip packages: {{install_pip}}" + ansible.builtin.pip: + name: 'yt-dlp' + state: "present" + virtualenv: '{{BINPROVIDERS.pip.virtualenv}}' + virtualenv_python: "{{BINPROVIDERS.pip.virtualenv_python}}" + virtualenv_site_packages: yes + + - name: Load YTDLP binary + import_role: + name: load_binary + vars: + name: ytdlp + + - name: Load ffmpeg binary + import_role: + name: load_binary + vars: + name: ffmpeg + version_cmd: 'ffmpeg -version' + + - name: Check that installed YT-DLP matches expected version + assert: + that: BINARIES.ytdlp.version is version(YTDLP_VERSION_MIN, '>=') + quiet: true + + - debug: + msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" diff --git a/archivebox/builtin_plugins/ytdlp/roles b/archivebox/builtin_plugins/ytdlp/roles new file mode 120000 index 00000000..e4109d37 --- /dev/null +++ b/archivebox/builtin_plugins/ytdlp/roles @@ -0,0 +1 @@ +../ansible/roles \ No newline at end of file diff --git a/archivebox/playbooks/install_all.yml b/archivebox/playbooks/install_all.yml old mode 100644 new mode 100755 index 5f7073e5..91cd9877 --- a/archivebox/playbooks/install_all.yml +++ b/archivebox/playbooks/install_all.yml @@ -1,3 +1,4 @@ +#!/usr/bin/env ansible-playbook --- - import_playbook: install_ytdlp.yml vars: @@ -7,6 +8,9 @@ vars: state: 'present' +- import_playbook: install_puppeteer.yml + vars: + state: 'present' - import_playbook: install_package.yml vars: diff --git a/archivebox/playbooks/install_npm.yml b/archivebox/playbooks/install_npm.yml old mode 100644 new mode 100755 index 08d60b6f..44cb25e7 --- a/archivebox/playbooks/install_npm.yml +++ b/archivebox/playbooks/install_npm.yml @@ -1,3 +1,4 @@ +#!/usr/bin/env ansible-playbook --- - import_playbook: install_package.yml @@ -71,7 +72,7 @@ installer_version: "{{BINARIES.npm.version}}" PATH: "{{LIB_DIR_NPM_BIN}}" when: BINPROVIDERS.npm is not defined - + - set_fact: BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(NODE_BINPROVIDERS) }}" cacheable: true diff --git a/archivebox/playbooks/install_package.yml b/archivebox/playbooks/install_package.yml old mode 100644 new mode 100755 index ef09fae1..6433bcb8 --- a/archivebox/playbooks/install_package.yml +++ b/archivebox/playbooks/install_package.yml @@ -1,3 +1,4 @@ +#!/usr/bin/env ansible-playbook --- - name: "Install using apt/brew/pkg/yum/etc." hosts: localhost diff --git a/archivebox/playbooks/install_pip.yml b/archivebox/playbooks/install_pip.yml old mode 100644 new mode 100755 index 421bcc33..c85ea33b --- a/archivebox/playbooks/install_pip.yml +++ b/archivebox/playbooks/install_pip.yml @@ -1,3 +1,4 @@ +#!/usr/bin/env ansible-playbook --- - import_playbook: load_binaries.yml @@ -64,7 +65,7 @@ installer_version: "{{BINARIES.pip.version}}" PATH: "{{LIB_DIR_PIP_BIN}}" when: BINPROVIDERS.pip is not defined - + - set_fact: BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(PIP_BINPROVIDERS) }}" cacheable: true diff --git a/archivebox/playbooks/install_puppeteer.yml b/archivebox/playbooks/install_puppeteer.yml old mode 100644 new mode 100755 index 9618e4cd..fd118a07 --- a/archivebox/playbooks/install_puppeteer.yml +++ b/archivebox/playbooks/install_puppeteer.yml @@ -1,71 +1,88 @@ +#!/usr/bin/env ansible-playbook --- -- import_playbook: install_npm.yml - vars: - packages: - - 'puppeteer' - - '@puppeteer/browsers' - state: 'latest' - - name: "Install puppeteer, puppeteer/browsers, and chrome" hosts: localhost gather_facts: no vars: - DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4' + DATA_DIR: '{{playbook_dir}}' LIB_DIR: '{{DATA_DIR}}/lib' - LIB_DIR_NPM: '{{LIB_DIR}}/npm' - LIB_DIR_NPM_BIN: '{{LIB_DIR_NPM}}/node_modules/.bin' LIB_DIR_BROWSERS: '{{LIB_DIR}}/browsers' - LIB_DIR_BIN: '{{LIB_DIR}}/bin' CHROME_RELEASE_CHANNEL: 'chrome@stable' CHROME_VERSION_MIN: '128.0.6613.137' - environment: - PATH: "{{LIB_DIR_NPM_BIN}}:{{ ansible_env.PATH }}" tasks: + - include_role: + name: setup_lib_npm + vars: + MIN_NODE_VERSION: '20.0.0' + MIN_NPM_VERSION: '10.0.0' + + - name: "Install npm packages: [puppeteer, @puppeteer/browsers]" + community.general.npm: + name: '{{item}}' + state: "present" + path: '{{BINPROVIDERS.npm.root_path}}' + loop: + - 'puppeteer' + - '@puppeteer/browsers' + - name: Make sure prerequisite folders exist file: - path: '{{item}}' + path: '{{LIB_DIR_BROWSERS}}' state: directory recurse: true + + - name: Load puppeteer binary from installed NPM package + include_role: + name: load_binaries + vars: + load_binaries: + - name: puppeteer + bin_name: 'puppeteer' + version_cmd: 'puppeteer --version' + PATH: '{{BINPROVIDERS.npm.PATH}}' + + - name: Load chrome binary from environment PATH + include_role: + name: load_binaries + vars: + load_binaries: + - name: 'chrome' + bin_name: '{{item}}' + PATH: '{{ansible_env.PATH}}' loop: - - '{{LIB_DIR_BROWSERS}}' - - '{{LIB_DIR_BIN}}' - - - name: Make sure Node and NPM are installed - assert: - that: - - BINARIES.node.version - - BINARIES.npm.version - quiet: true - - - name: Get installed puppeteer version - command: 'puppeteer --version' - register: PUPPETEER_VERSION_FULL - changed_when: False + - chrome + - chrome-browser + - chromium + - chromium-browser + - google-chrome + - google-chrome-browser + - google-chrome-stable + - google-chrome-beta + - google-chrome-canary + - google-chrome-unstable + - google-chrome-dev ################################################################################### - name: Install Chrome browser using puppeteer/browsers command: 'npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}} --path {{LIB_DIR_BROWSERS}}' register: CHROME_VERSION_FULL - # -> 'chrome@128.0.6613.137 /data/lib/browsers/chrome/linux_arm-128.0.6613.138/chrome-linux-arm64/...' + environment: + PATH: "{{BINPROVIDERS.npm.PATH}}:{{ ansible_env.PATH }}" changed_when: CHROME_VERSION_MIN not in CHROME_VERSION_FULL.stdout + when: BINARIES.chrome is not defined + # -> 'chrome@128.0.6613.137 /data/lib/browsers/chrome/linux_arm-128.0.6613.138/chrome-linux-arm64/...' ################################################################################### - set_fact: - PUPPETEER_ABSPATH: "{{LIB_DIR_NPM_BIN}}/puppeteer" - PUPPETEER_VERSION: "{{ PUPPETEER_VERSION_FULL.stdout_lines|first }}" CHROME_ABSPATH: "{{ CHROME_VERSION_FULL.stdout_lines|last|split(' ', 1)|last }}" CHROME_VERSION: "{{ CHROME_VERSION_FULL.stdout_lines|last|split('@', 1)|last|split(' ', 1)|first }}" - - - name: Check that installed Chrome matches expected version - assert: - that: CHROME_VERSION_MIN is version(CHROME_VERSION, '>=') - quiet: true + when: BINARIES.chrome is not defined - name: Create ./bin/chrome symlink to ./browsers/chrome/... binary copy: content: | #!/bin/bash - exec '{{CHROME_ABSPATH}}' "$@" + exec '{{CHROME_ABSPATH|default(BINARIES.chrome.abspath)}}' "$@" dest: "{{LIB_DIR_BIN}}/chrome" - file: path: "{{LIB_DIR_BIN}}/chrome" @@ -74,18 +91,22 @@ ################################################################################### - set_fact: - PUPPETEER_DEPENDENCIES: - node: "{{BINARIES.node}}" - npm: "{{BINARIES.npm}}" PUPPETEER_BINARIES: - puppeteer: - abspath: "{{PUPPETEER_ABSPATH}}" - version: "{{PUPPETEER_VERSION}}" chrome: - abspath: "{{CHROME_ABSPATH}}" - version: "{{CHROME_VERSION}}" + bin_name: 'chrome' + abspath: "{{CHROME_ABSPATH|default(BINARIES.chrome.abspath)}}" + version: "{{CHROME_VERSION|default(BINARIES.chrome.version)}}" + symlink: "{{LIB_DIR_BIN}}/chrome" + version_cmd: "chrome --version" + + - name: Check that installed Chrome matches expected version + assert: + that: PUPPETEER_BINARIES.chrome.version is version(CHROME_VERSION_MIN, '>=') + quiet: true + - set_fact: BINARIES: "{{ BINARIES | combine(PUPPETEER_BINARIES) }}" + cacheable: true - debug: msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" diff --git a/archivebox/playbooks/install_singlefile.yml b/archivebox/playbooks/install_singlefile.yml old mode 100644 new mode 100755 index 5e4d8c0f..9e33e32f --- a/archivebox/playbooks/install_singlefile.yml +++ b/archivebox/playbooks/install_singlefile.yml @@ -1,16 +1,8 @@ +#!/usr/bin/env ansible-playbook --- -- import_playbook: install_npm.yml - vars: - install_npm: - singlefile: {packages: ['single-file-cli@1.1.54']} - state: 'present' - -- import_playbook: load_binaries.yml - vars: - load_binaries: - singlefile: {bin_name: 'single-file', version_cmd: 'single-file --version', PATH: '{{BINPROVIDERS.npm.PATH}}'} - +- import_playbook: ../puppeteer/install_puppeteer.yml + when: BINARIES.chrome is not defined - name: "Install Singlefile" hosts: localhost @@ -18,12 +10,26 @@ vars: SINGLEFILE_VERSION_EXACT: '1.1.54' tasks: - - name: Make sure Node and NPM are installed - assert: - that: - - BINARIES.node.version - - BINARIES.npm.version - quiet: true + - include_role: + name: setup_lib_npm + vars: + MIN_NODE_VERSION: '20.0.0' + MIN_NPM_VERSION: '10.0.0' + + - name: "Install npm packages: [single-file-cli]" + community.general.npm: + name: 'single-file-cli@1.1.54' + state: "present" + path: '{{BINPROVIDERS.npm.root_path}}' + + - name: Load single-file binary from installed NPM package + include_role: + name: load_binary + vars: + name: singlefile + bin_name: 'single-file' + version_cmd: 'single-file --version' + PATH: '{{BINPROVIDERS.npm.PATH}}' - name: Check that installed Singlefile version matches expected version assert: diff --git a/archivebox/playbooks/install_ytdlp.yml b/archivebox/playbooks/install_ytdlp.yml old mode 100644 new mode 100755 index 16881037..b9c7eba4 --- a/archivebox/playbooks/install_ytdlp.yml +++ b/archivebox/playbooks/install_ytdlp.yml @@ -1,37 +1,24 @@ +#!/usr/bin/env ansible-playbook --- -- import_playbook: install_package.yml - vars: - install_packages: - ffmpeg: {packages: ['ffmpeg']} - state: 'present' - -- import_playbook: install_pip.yml - vars: - install_pip: - ytdlp: {packages: ['yt-dlp']} - state: 'present' - -- import_playbook: load_binaries.yml - vars: - load_binaries: - ffmpeg: {bin_name: 'ffmpeg', version_cmd: 'ffmpeg -version'} - ytdlp: {bin_name: 'yt-dlp', version_cmd: 'yt-dlp --version'} - - - name: "Install YT-DLP" hosts: localhost gather_facts: no vars: YTDLP_VERSION_MIN: '2024.8.6' tasks: - - name: Make sure Python and Pip are installed - assert: - that: - - BINARIES.python.version - - BINARIES.pip.version - - BINARIES.ffmpeg.version - - BINARIES.ytdlp.version - quiet: true + - include_role: + name: setup_lib_pip + vars: + MIN_PYTHON_VERSION: '3.10.0' + MIN_PIP_VERSION: '22.0' + + - name: "Install pip packages: {{install_pip}}" + ansible.builtin.pip: + name: 'yt-dlp' + state: "present" + virtualenv: '{{LIB_DIR_PIP}}/venv' + virtualenv_python: "{{BINARIES.python.abspath}}" + virtualenv_site_packages: yes - name: Check that installed YT-DLP matches expected version assert: @@ -40,3 +27,23 @@ - debug: msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" + + + +# - import_playbook: install_package.yml +# vars: +# install_packages: +# ffmpeg: {packages: ['ffmpeg']} +# state: 'present' + +# - import_playbook: install_pip.yml +# vars: +# install_pip: +# ytdlp: {packages: ['yt-dlp']} +# state: 'present' + +# - import_playbook: load_binaries.yml +# vars: +# load_binaries: +# ffmpeg: {bin_name: 'ffmpeg', version_cmd: 'ffmpeg -version'} +# ytdlp: {bin_name: 'yt-dlp', version_cmd: 'yt-dlp --version'} diff --git a/archivebox/playbooks/load_binaries.yml b/archivebox/playbooks/load_binaries.yml old mode 100644 new mode 100755 index eb0239b9..c8fb8b40 --- a/archivebox/playbooks/load_binaries.yml +++ b/archivebox/playbooks/load_binaries.yml @@ -1,3 +1,4 @@ +#!/usr/bin/env ansible-playbook --- - name: Get binary abspaths and versions and add to BINARIES hosts: localhost