consolidate ansible setup into roles dir

archivebox/builtin_plugins/ansible/roles/install_packages/tasks/main.yml

@@ -0,0 +1,58 @@
+#!/usr/bin/env ansible-playbook
+---
+# - name: "Install using apt/brew/pkg/yum/etc."
+#   hosts: localhost
+#   gather_facts: no
+#   vars:
+#     DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4'
+#     LIB_DIR: '{{DATA_DIR}}/lib'
+#     LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+#     state: 'present'
+#     install_packages: {}
+#       # bash:
+#       #   packages: ['bash']
+#     PACKAGE_BINPROVIDERS: {}
+#     PACKAGE_BINARIES: {}
+#     BINPROVIDERS: {}
+#   tasks:
+# - package: update_cache=yes
+  # when: ansible_facts['os_family'] == "Debian"
+
+- name: Make sure lib folders exist
+  file:
+    path: '{{LIB_DIR_BIN}}'
+    state: directory
+    recurse: true
+  when: BINPROVIDERS.ansible_package is not defined
+
+- name: Get ansible binary abspath
+  command: 'which ansible'
+  register: ANSIBLE_INSTALLER_ABSPATH_FULL
+  changed_when: false
+  when: BINPROVIDERS.ansible_package is not defined
+
+###################################################################################
+- name: "Install system packages: {{all_packages}}"
+  ansible.builtin.package:
+    name: "{{item}}"
+    state: "{{state}}"
+  loop: "{{all_packages}}"
+
+###################################################################################
+
+- set_fact:
+    PACKAGE_BINPROVIDERS:
+      ansible_package:
+        installer_abspath: "{{ANSIBLE_INSTALLER_ABSPATH_FULL.stdout}}"
+        installer_version: "{{ansible_version.full}}"
+        PATH: "/opt/homebrew/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+  when: BINPROVIDERS.ansible_package is not defined
+
+- set_fact:
+    BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(PACKAGE_BINPROVIDERS) }}"
+    cacheable: true
+  when: BINPROVIDERS.ansible_package is not defined
+
+- debug:
+    msg: "{{ {'BINPROVIDERS': BINPROVIDERS} }}"
+

archivebox/builtin_plugins/ansible/roles/install_packages/vars/main.yml

@@ -0,0 +1,5 @@
+DATA_DIR: '{{playbook_dir}}'
+LIB_DIR: '{{DATA_DIR}}/lib'
+LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+state: present
+all_packages: []

archivebox/builtin_plugins/ansible/roles/load_binary/meta/argument_specs.yml

@@ -0,0 +1,31 @@
+---
+argument_specs:
+  main:
+    short_description: Load a specified binary from the environment/PATH into BINARIES fact.
+
+    options:
+
+      name:
+        type: "str"
+        required: true
+        description: "A string key for the binary"
+
+      bin_name:
+        type: "str"
+        required: false
+        description: "The basename of the binary file (optional, defaults to name)"
+
+      abspath:
+        type: "str"
+        required: false
+        description: "An absolute path to the binary (overrides any auto-detected one)"
+
+      version_cmd:
+        type: "str"
+        required: false
+        description: "The command to run to get the binary's version (optional, defaults to $ <bin_name> --version)"
+
+      PATH:
+        type: "str"
+        required: false
+        description: "The PATH to search for the binary (optional, defaults to environment $PATH)"

archivebox/builtin_plugins/ansible/roles/load_binary/tasks/main.yml

@@ -0,0 +1,73 @@
+
+---
+- name: Make sure ./data/lib/bin folder exists
+  file:
+    path: '{{LIB_DIR_BIN}}'
+    state: directory
+    recurse: true
+
+###################################################################################
+
+- set_fact:
+    PATH: "{{PATH or DEFAULT_PATH}}"
+    bin_name: "{{bin_name or name}}"
+    BINARY_ABSPATH: null
+    BINARY_VERSION: null
+
+- name: 'Get installed binary abspath: {{name}}'
+  command: 'env PATH="{{PATH}}:$PATH" which {{bin_name}}'
+  register: BINARY_ABSPATH
+  changed_when: BINARIES[name].abspath|default('NO VERSION FOUND') not in BINARY_ABSPATH.stdout
+
+- set_fact:
+    bin_name: "{{BINARY_ABSPATH.stdout|basename or bin_name}}"
+- set_fact:
+    version_cmd: "{{version_cmd or (bin_name + ' --version')}}"
+
+- name: 'Get installed binary version: {{name}}'
+  command: 'env PATH="{{PATH}}:$PATH" {{version_cmd}}'
+  register: BINARY_VERSION
+  changed_when: BINARIES[name].version|default('NO VERSION FOUND') not in BINARY_VERSION.stdout
+
+
+- name: 'Updating BINARIES with loaded abspaths & versions: {{name}}'
+  set_fact:
+    BINARIES: "{{      BINARIES
+      | default({})
+      | combine({
+          name: {
+            'name': name,
+            'bin_name': bin_name,
+            'version_cmd': version_cmd,
+            'symlink': LIB_DIR_BIN + '/' + name,
+            'abspath': BINARY_ABSPATH.stdout or abspath or None,
+            'version': BINARY_VERSION.stdout_lines|first|regex_replace('^.*?([\\d+\\.]+).*$', '\\1') or version or None,
+            'version_stdout': BINARY_VERSION.stdout or BINARY_VERSION.stderr,
+            'PATH': BINARY_ABSPATH.stdout|dirname or PATH,
+          },
+      })
+    }}"
+    cacheable: true
+  when: BINARY_ABSPATH.stdout and BINARY_VERSION.stdout
+
+- name: 'Symlink installed binary into lib bin folder: {{name}}'
+  file:
+    src: "{{ BINARY_ABSPATH.stdout }}"
+    dest: "{{ LIB_DIR_BIN }}/{{ name }}"
+    state: link
+    force: true
+  when: BINARY_ABSPATH.stdout and BINARY_VERSION.stdout
+  changed_when: False
+
+- debug:
+    msg:
+      - '{{BINARIES}}'
+
+- name: Unset variables
+  set_fact:
+    name:
+    bin_name:
+    version_cmd:
+    PATH:
+    BINARY_ABSPATH:
+    BINARY_VERSION:

archivebox/builtin_plugins/ansible/roles/load_binary/vars/main.yml

@@ -0,0 +1,11 @@
+DATA_DIR: '{{playbook_dir}}'
+LIB_DIR: '{{DATA_DIR}}/lib'
+LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+DEFAULT_PATH: /bin
+
+name:
+bin_name:
+version_cmd:
+PATH:
+abspath:
+version:

archivebox/builtin_plugins/ansible/roles/setup_lib_npm/meta/argument_specs.yml

@@ -0,0 +1,31 @@
+---
+argument_specs:
+  main:
+    short_description: Main entry point for the npm role
+
+    options:
+    
+      state:
+        type: "str"
+        required: false
+        default: 'present'
+        description:
+          - "The desired state: present | latest"
+
+      npm_packages:
+        type: "list"
+        elements: "dict"
+        required: false
+        default: []
+        description: "A list of dicts with a defined structure and with default a value."
+        options:
+          key:
+            type: "str"
+            required: true
+            description: "A string name for the dependency"
+
+          packages:
+            type: "list"
+            elements: "str"
+            required: true
+            description: "What npm packages to install for the given dependency."

archivebox/builtin_plugins/ansible/roles/setup_lib_npm/tasks/main.yml

@@ -0,0 +1,59 @@
+---
+- name: Make sure lib folders exist
+  file:
+    path: '{{item}}'
+    state: directory
+    recurse: true
+  loop:
+    - '{{LIB_DIR_NPM_BIN}}'
+    - '{{LIB_DIR_BIN}}'
+
+- name: "Install system packages: [node, npm]"
+  ansible.builtin.package:
+    name: "node"
+    state: "present"
+
+- name: Load NPM and Node binaries
+  include_role:
+    name: load_binary
+  vars:
+    name: '{{item}}'
+  loop:
+    - node
+    - npm
+
+- name: Check that installed Node version matches expected version
+  assert:
+    that:
+      - BINARIES.node.version is version(MIN_NODE_VERSION, '>=')
+      - BINARIES.npm.version is version(MIN_NPM_VERSION, '>=')
+    quiet: true
+
+###################################################################################
+
+
+# - name: "Install npm packages: {{install_npm}}"
+#   community.general.npm:
+#     name: '{{item}}'
+#     state: "{{state}}"
+#     path: '{{LIB_DIR_NPM}}'
+#   loop: "{{install_npm|dictsort|map(attribute='1')|map(attribute='packages')|flatten}}"
+
+###################################################################################
+
+
+###################################################################################
+- set_fact:
+    NODE_BINPROVIDERS:
+      npm:
+        installer_abspath: "{{BINARIES.npm.abspath}}"
+        installer_version: "{{BINARIES.npm.version}}"
+        PATH: "{{LIB_DIR_NPM_BIN}}"
+        lib_dir_npm: "{{LIB_DIR_NPM}}"
+
+- set_fact:
+    BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(NODE_BINPROVIDERS) }}"
+    cacheable: true
+
+- debug:
+    msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/builtin_plugins/ansible/roles/setup_lib_npm/vars/main.yml

@@ -0,0 +1,7 @@
+DATA_DIR: '{{playbook_dir}}'
+LIB_DIR: '{{DATA_DIR}}/lib'
+LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+LIB_DIR_NPM: '{{LIB_DIR}}/npm'
+LIB_DIR_NPM_BIN: '{{LIB_DIR_NPM}}/node_modules/.bin'
+MIN_NODE_VERSION: '20.0.0'
+MIN_NPM_VERSION: '10.0.0'

archivebox/builtin_plugins/ansible/roles/setup_lib_pip/meta/argument_specs.yml

@@ -0,0 +1,31 @@
+---
+argument_specs:
+  main:
+    short_description: Main entry point for the npm role
+
+    options:
+    
+      state:
+        type: "str"
+        required: false
+        default: 'present'
+        description:
+          - "The desired state: present | latest"
+
+      npm_packages:
+        type: "list"
+        elements: "dict"
+        required: false
+        default: []
+        description: "A list of dicts with a defined structure and with default a value."
+        options:
+          key:
+            type: "str"
+            required: true
+            description: "A string name for the dependency"
+
+          packages:
+            type: "list"
+            elements: "str"
+            required: true
+            description: "What npm packages to install for the given dependency."

archivebox/builtin_plugins/ansible/roles/setup_lib_pip/tasks/main.yml

@@ -0,0 +1,59 @@
+---
+- name: Make sure lib folders exist
+  file:
+    path: '{{item}}'
+    state: directory
+    recurse: true
+  loop:
+    - '{{LIB_DIR_PIP}}'
+    - '{{LIB_DIR_BIN}}'
+  when: BINPROVIDERS.pip is not defined
+
+- name: Load Python and Pip binaries
+  import_role:
+    name: load_binary
+  vars:
+    name: '{{item}}'
+  loop:
+    - python
+    - pip
+  when: BINARIES.python is not defined or BINARIES.pip is not defined
+
+- assert:
+    that:
+      - BINARIES.python.version is version(MIN_PYTHON_VERSION, '>=')
+      - BINARIES.pip.version is version(MIN_PIP_VERSION, '>=')
+    quiet: true
+  when: BINPROVIDERS.pip is not defined
+
+###################################################################################
+
+
+# - name: "Install pip packages: {{install_pip}}"
+#   ansible.builtin.pip:
+#     name: '{{item}}'
+#     state: "{{state}}"
+#     virtualenv: '{{LIB_DIR_PIP}}/venv'
+#     virtualenv_python: "{{BINARIES.python.abspath}}"
+#     virtualenv_site_packages: yes
+#   loop: "{{install_pip|dictsort|map(attribute='1')|map(attribute='packages')|flatten}}"
+
+
+###################################################################################
+- set_fact:
+    PIP_BINPROVIDERS:
+      pip:
+        installer_abspath: "{{BINARIES.pip.abspath}}"
+        installer_version: "{{BINARIES.pip.version}}"
+        PATH: "{{LIB_DIR_PIP_BIN}}"
+        virtualenv: "{{LIB_DIR_PIP}}/venv"
+        virtualenv_python: "{{BINARIES.python.abspath}}"
+  when: BINPROVIDERS.pip is not defined
+
+- set_fact:
+    BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(PIP_BINPROVIDERS) }}"
+    cacheable: true
+  when: BINPROVIDERS.pip is not defined
+
+- debug:
+    msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/builtin_plugins/ansible/roles/setup_lib_pip/vars/main.yml

@@ -0,0 +1,7 @@
+DATA_DIR: '{{playbook_dir}}'
+LIB_DIR: '{{DATA_DIR}}/lib'
+LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+LIB_DIR_PIP: '{{LIB_DIR}}/pip'
+LIB_DIR_PIP_BIN: '{{LIB_DIR_PIP}}/venv/bin'
+MIN_PYTHON_VERSION: '3.10.0'
+MIN_PIP_VERSION: '22.0'

archivebox/builtin_plugins/puppeteer/apps.py

@@ -0,0 +1,85 @@
+from typing import List, Optional
+
+from django.conf import settings
+
+# Depends on other PyPI/vendor packages:
+from pydantic import InstanceOf, Field
+from pydantic_pkgr import BinProvider, BinName
+
+# Depends on other Django apps:
+from plugantic.base_plugin import BasePlugin
+from plugantic.base_configset import BaseConfigSet, ConfigSectionName
+from plugantic.base_binary import BaseBinary, env
+# from plugantic.base_extractor import BaseExtractor
+# from plugantic.base_queue import BaseQueue
+from plugantic.base_hook import BaseHook
+from plugantic.ansible_utils import run_playbook
+
+# Depends on Other Plugins:
+from builtin_plugins.npm.apps import npm
+
+
+###################### Config ##########################
+
+
+class PuppeteerDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    PUPPETEER_BINARY: str = Field(default='wget')
+    PUPPETEER_ARGS: Optional[List[str]] = Field(default=None)
+    PUPPETEER_EXTRA_ARGS: List[str] = []
+    PUPPETEER_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
+
+class PuppeteerConfigs(PuppeteerDependencyConfigs):
+    # section: ConfigSectionName = 'ALL_CONFIGS'
+    pass
+
+DEFAULT_GLOBAL_CONFIG = {
+}
+
+PUPPETEER_CONFIG = PuppeteerConfigs(**DEFAULT_GLOBAL_CONFIG)
+
+
+INSTALL_BIN = './install_puppeteer.yml'
+
+
+class ChromeBinary(BaseBinary):
+    name: BinName = 'chrome'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [npm, env]
+
+    
+    def install(self, *args, quiet=False) -> "ChromeBinary":
+        
+        install_playbook = self.plugin_dir / 'install_puppeteer.yml'
+        
+        chrome_bin = run_playbook(install_playbook, data_dir=settings.CONFIG.OUTPUT_DIR, quiet=quiet).BINARIES.chrome
+
+        return self.__class__.model_validate(
+            {
+                **self.model_dump(),
+                "loaded_abspath": chrome_bin.symlink,
+                "loaded_version": chrome_bin.version,
+                "loaded_binprovider": env,
+                "binproviders_supported": self.binproviders_supported,
+            }
+        )
+        
+
+CHROME_BINARY = ChromeBinary()
+
+PLUGIN_BINARIES = [CHROME_BINARY]
+
+class PuppeteerPlugin(BasePlugin):
+    app_label: str ='puppeteer'
+    verbose_name: str = 'SingleFile'
+
+    hooks: List[InstanceOf[BaseHook]] = [
+        PUPPETEER_CONFIG,
+        CHROME_BINARY,
+    ]
+
+
+
+PLUGIN = PuppeteerPlugin()
+PLUGIN.register(settings)
+DJANGO_APP = PLUGIN.AppConfig

archivebox/builtin_plugins/puppeteer/install_puppeteer.yml

@@ -0,0 +1,127 @@
+#!/usr/bin/env ansible-playbook
+---
+- name: "Install puppeteer, puppeteer/browsers, and chrome"
+  hosts: localhost
+  gather_facts: true
+  vars:
+    LIB_DIR: '{{DATA_DIR}}/lib'
+    LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+    LIB_DIR_BROWSERS: '{{LIB_DIR}}/browsers'
+    CHROME_RELEASE_CHANNEL: 'chrome@stable'
+    CHROME_VERSION_MIN: '128.0.6613.137'
+  tasks:
+    - include_role:
+        name: setup_lib_npm
+      vars:
+        MIN_NODE_VERSION: '20.0.0'
+        MIN_NPM_VERSION: '10.0.0'
+
+    - name: "Install npm packages: [puppeteer, @puppeteer/browsers]"
+      community.general.npm:
+        name: '{{item}}'
+        state: "present"
+        path: '{{BINPROVIDERS.npm.lib_dir_npm}}'
+      loop:
+        - 'puppeteer'
+        - '@puppeteer/browsers'
+      
+    - name: Make sure prerequisite folders exist
+      file:
+        path: '{{LIB_DIR_BROWSERS}}'
+        state: directory
+        recurse: true
+
+    - name: Load puppeteer binary from installed NPM package
+      include_role:
+        name: load_binary
+      vars:
+        name: puppeteer
+        PATH: '{{BINPROVIDERS.npm.PATH}}'
+
+    - name: Load chrome binaries from environment PATH
+      include_tasks: roles/load_binary/tasks/main.yml
+      vars:
+        name: 'chrome'
+        bin_name: '{{chrome_executable}}'
+        PATH: '{{ansible_env.PATH}}'
+      with_items:
+        - chrome
+        # - chrome-browser
+        # - chromium
+        # - chromium-browser
+        # - google-chrome
+        # - google-chrome-browser
+        # - google-chrome-stable
+        # - google-chrome-beta
+        # - google-chrome-canary
+        # - google-chrome-unstable
+        # - google-chrome-dev
+      loop_control:
+        loop_var: chrome_executable
+        # break_when: 
+        #   - BINARIES.chrome.version|default('')
+
+    # - debug:
+    #     msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"
+
+
+    ###################################################################################
+    - name: 'Install Chrome browser: npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}}'
+      command: 'npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}} --path {{LIB_DIR_BROWSERS}}'
+      register: CHROME_VERSION_FULL
+      environment:
+        PATH: "{{BINPROVIDERS.npm.PATH}}:{{ ansible_env.PATH }}"
+      changed_when: CHROME_VERSION_MIN not in CHROME_VERSION_FULL.stdout
+      when: not BINARIES.chrome.version|default('')
+      #         -> 'chrome@128.0.6613.137 /data/lib/browsers/chrome/linux_arm-128.0.6613.138/chrome-linux-arm64/...'
+
+    ###################################################################################
+    - name: Parse Chrome version and abspath from npx @puppeteer/browsers install output
+      set_fact:
+        CHROME_ABSPATH: "{{ CHROME_VERSION_FULL.stdout_lines|last|split(' ', 1)|last }}"
+        CHROME_VERSION: "{{ CHROME_VERSION_FULL.stdout_lines|last|split('@', 1)|last|split(' ', 1)|first }}"
+      when: not BINARIES.chrome.version|default('')
+
+    - name: Create ./bin/chrome symlink to ./browsers/chrome/... binary
+      copy:
+        content: |
+                 #!/bin/bash
+                 exec '{{CHROME_ABSPATH|default(BINARIES.chrome.abspath)}}' "$@"
+        dest: "{{LIB_DIR_BIN}}/chrome"
+      changed_when: False
+    
+    - name: Ensure ./bin/chrome symlink is executable
+      file:
+        path: "{{LIB_DIR_BIN}}/chrome"
+        mode: u+rx,g-rx,o-rwx
+        state: 'file'
+      changed_when: False
+
+    ###################################################################################
+    - set_fact:
+        PUPPETEER_BINARIES:
+          chrome:
+            bin_name: 'chrome'
+            abspath: "{{CHROME_ABSPATH|default(BINARIES.chrome.abspath) or None}}"
+            version: "{{CHROME_VERSION|default(BINARIES.chrome.version) or None}}"
+            symlink: "{{LIB_DIR_BIN}}/chrome"
+            version_cmd: "chrome --version"
+            binprovider: 'puppeteer'
+        PUPPETEER_BINPROVIDERS:
+          puppeteer:
+            installer_abspath: "{{BINARIES.puppeteer.abspath}}"
+            installer_version: "{{BINARIES.puppeteer.version}}"
+            PATH: "{{LIB_DIR_BIN}}"
+            lib_dir_browsers: "{{LIB_DIR_BROWSERS}}"
+
+    - name: Check that installed Chrome matches expected version
+      assert:
+        that: PUPPETEER_BINARIES.chrome.version is version(CHROME_VERSION_MIN, '>=')
+        quiet: true
+    
+    - set_fact:
+        BINARIES: "{{ BINARIES | combine(PUPPETEER_BINARIES) }}"
+        cacheable: true
+
+    - debug:
+        msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/builtin_plugins/puppeteer/roles

@@ -0,0 +1 @@
+../ansible/roles

archivebox/builtin_plugins/singlefile/install_singlefile.yml

@@ -0,0 +1 @@
+../../playbooks/install_singlefile.yml

archivebox/builtin_plugins/singlefile/roles

@@ -0,0 +1 @@
+../ansible/roles

archivebox/builtin_plugins/ytdlp/install_ytdlp.yml

@@ -0,0 +1,42 @@
+#!/usr/bin/env ansible-playbook
+---
+- name: "Install YT-DLP"
+  hosts: localhost
+  gather_facts: no
+  vars:
+    YTDLP_VERSION_MIN: '2024.8.6'
+  tasks:
+    - include_role:
+        name: setup_lib_pip
+      vars:
+        MIN_PYTHON_VERSION: '3.10.0'
+        MIN_PIP_VERSION: '22.0'
+    
+    - name: "Install pip packages: {{install_pip}}"
+      ansible.builtin.pip:
+        name: 'yt-dlp'
+        state: "present"
+        virtualenv: '{{BINPROVIDERS.pip.virtualenv}}'
+        virtualenv_python: "{{BINPROVIDERS.pip.virtualenv_python}}"
+        virtualenv_site_packages: yes
+
+    - name: Load YTDLP binary
+      import_role:
+        name: load_binary
+      vars:
+        name: ytdlp
+
+    - name: Load ffmpeg binary
+      import_role:
+        name: load_binary
+      vars:
+        name: ffmpeg
+        version_cmd: 'ffmpeg -version'
+
+    - name: Check that installed YT-DLP matches expected version
+      assert:
+        that: BINARIES.ytdlp.version is version(YTDLP_VERSION_MIN, '>=')
+        quiet: true
+
+    - debug:
+        msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/builtin_plugins/ytdlp/roles

@@ -0,0 +1 @@
+../ansible/roles

archivebox/playbooks/install_all.yml

@@ -1,3 +1,4 @@
+#!/usr/bin/env ansible-playbook
 ---
 - import_playbook: install_ytdlp.yml
   vars:
@@ -7,6 +8,9 @@
   vars:
     state: 'present'
 
+- import_playbook: install_puppeteer.yml
+  vars:
+    state: 'present'
 
 - import_playbook: install_package.yml
   vars:

archivebox/playbooks/install_npm.yml

@@ -1,3 +1,4 @@
+#!/usr/bin/env ansible-playbook
 ---
 
 - import_playbook: install_package.yml

archivebox/playbooks/install_package.yml

@@ -1,3 +1,4 @@
+#!/usr/bin/env ansible-playbook
 ---
 - name: "Install using apt/brew/pkg/yum/etc."
   hosts: localhost

archivebox/playbooks/install_pip.yml

@@ -1,3 +1,4 @@
+#!/usr/bin/env ansible-playbook
 ---
 
 - import_playbook: load_binaries.yml

archivebox/playbooks/install_puppeteer.yml

@@ -1,71 +1,88 @@
+#!/usr/bin/env ansible-playbook
 ---
-- import_playbook: install_npm.yml
-  vars:
-    packages:
-      - 'puppeteer'
-      - '@puppeteer/browsers'
-    state: 'latest'
-
 - name: "Install puppeteer, puppeteer/browsers, and chrome"
   hosts: localhost
   gather_facts: no
   vars:
-    DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4'
+    DATA_DIR: '{{playbook_dir}}'
     LIB_DIR: '{{DATA_DIR}}/lib'
-    LIB_DIR_NPM: '{{LIB_DIR}}/npm'
-    LIB_DIR_NPM_BIN: '{{LIB_DIR_NPM}}/node_modules/.bin'
     LIB_DIR_BROWSERS: '{{LIB_DIR}}/browsers'
-    LIB_DIR_BIN: '{{LIB_DIR}}/bin'
     CHROME_RELEASE_CHANNEL: 'chrome@stable'
     CHROME_VERSION_MIN: '128.0.6613.137'
-  environment:
-    PATH: "{{LIB_DIR_NPM_BIN}}:{{ ansible_env.PATH }}"
   tasks:
+    - include_role:
+        name: setup_lib_npm
+      vars:
+        MIN_NODE_VERSION: '20.0.0'
+        MIN_NPM_VERSION: '10.0.0'
+
+    - name: "Install npm packages: [puppeteer, @puppeteer/browsers]"
+      community.general.npm:
+        name: '{{item}}'
+        state: "present"
+        path: '{{BINPROVIDERS.npm.root_path}}'
+      loop:
+        - 'puppeteer'
+        - '@puppeteer/browsers'
+      
     - name: Make sure prerequisite folders exist
       file:
-        path: '{{item}}'
+        path: '{{LIB_DIR_BROWSERS}}'
         state: directory
         recurse: true
+
+    - name: Load puppeteer binary from installed NPM package
+      include_role:
+        name: load_binaries
+      vars:
+        load_binaries:
+          - name: puppeteer
+            bin_name: 'puppeteer'
+            version_cmd: 'puppeteer --version'
+            PATH: '{{BINPROVIDERS.npm.PATH}}'
+
+    - name: Load chrome binary from environment PATH
+      include_role:
+        name: load_binaries
+      vars:
+        load_binaries:
+          - name: 'chrome'
+            bin_name: '{{item}}'
+            PATH: '{{ansible_env.PATH}}'
       loop:
-        - '{{LIB_DIR_BROWSERS}}'
+        - chrome
-        - '{{LIB_DIR_BIN}}'
+        - chrome-browser
-
+        - chromium
-    - name: Make sure Node and NPM are installed
+        - chromium-browser
-      assert:
+        - google-chrome
-        that:
+        - google-chrome-browser
-          - BINARIES.node.version
+        - google-chrome-stable
-          - BINARIES.npm.version
+        - google-chrome-beta
-        quiet: true
+        - google-chrome-canary
-
+        - google-chrome-unstable
-    - name: Get installed puppeteer version
+        - google-chrome-dev
-      command: 'puppeteer --version'
-      register: PUPPETEER_VERSION_FULL
-      changed_when: False
 
     ###################################################################################
     - name: Install Chrome browser using puppeteer/browsers
       command: 'npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}} --path {{LIB_DIR_BROWSERS}}'
       register: CHROME_VERSION_FULL
-      #         -> 'chrome@128.0.6613.137 /data/lib/browsers/chrome/linux_arm-128.0.6613.138/chrome-linux-arm64/...'
+      environment:
+        PATH: "{{BINPROVIDERS.npm.PATH}}:{{ ansible_env.PATH }}"
       changed_when: CHROME_VERSION_MIN not in CHROME_VERSION_FULL.stdout
+      when: BINARIES.chrome is not defined
+      #         -> 'chrome@128.0.6613.137 /data/lib/browsers/chrome/linux_arm-128.0.6613.138/chrome-linux-arm64/...'
 
     ###################################################################################
     - set_fact:
-        PUPPETEER_ABSPATH: "{{LIB_DIR_NPM_BIN}}/puppeteer"
-        PUPPETEER_VERSION: "{{ PUPPETEER_VERSION_FULL.stdout_lines|first }}"
         CHROME_ABSPATH: "{{ CHROME_VERSION_FULL.stdout_lines|last|split(' ', 1)|last }}"
         CHROME_VERSION: "{{ CHROME_VERSION_FULL.stdout_lines|last|split('@', 1)|last|split(' ', 1)|first }}"
-
+      when: BINARIES.chrome is not defined
-    - name: Check that installed Chrome matches expected version
-      assert:
-        that: CHROME_VERSION_MIN is version(CHROME_VERSION, '>=')
-        quiet: true
 
     - name: Create ./bin/chrome symlink to ./browsers/chrome/... binary
       copy:
         content: |
                  #!/bin/bash
-                 exec '{{CHROME_ABSPATH}}' "$@"
+                 exec '{{CHROME_ABSPATH|default(BINARIES.chrome.abspath)}}' "$@"
         dest: "{{LIB_DIR_BIN}}/chrome"
     - file:
         path: "{{LIB_DIR_BIN}}/chrome"
@@ -74,18 +91,22 @@
 
     ###################################################################################
     - set_fact:
-        PUPPETEER_DEPENDENCIES:
-          node: "{{BINARIES.node}}"
-          npm: "{{BINARIES.npm}}"
         PUPPETEER_BINARIES:
-          puppeteer:
-            abspath: "{{PUPPETEER_ABSPATH}}"
-            version: "{{PUPPETEER_VERSION}}"
           chrome:
-            abspath: "{{CHROME_ABSPATH}}"
+            bin_name: 'chrome'
-            version: "{{CHROME_VERSION}}"
+            abspath: "{{CHROME_ABSPATH|default(BINARIES.chrome.abspath)}}"
+            version: "{{CHROME_VERSION|default(BINARIES.chrome.version)}}"
+            symlink: "{{LIB_DIR_BIN}}/chrome"
+            version_cmd: "chrome --version"
+
+    - name: Check that installed Chrome matches expected version
+      assert:
+        that: PUPPETEER_BINARIES.chrome.version is version(CHROME_VERSION_MIN, '>=')
+        quiet: true
+    
     - set_fact:
         BINARIES: "{{ BINARIES | combine(PUPPETEER_BINARIES) }}"
+        cacheable: true
 
     - debug:
         msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/playbooks/install_singlefile.yml

@@ -1,16 +1,8 @@
+#!/usr/bin/env ansible-playbook
 ---
 
-- import_playbook: install_npm.yml
+- import_playbook: ../puppeteer/install_puppeteer.yml
-  vars:
+  when: BINARIES.chrome is not defined
-    install_npm:
-      singlefile: {packages: ['single-file-cli@1.1.54']}
-    state: 'present'
-
-- import_playbook: load_binaries.yml
-  vars:
-    load_binaries:
-      singlefile: {bin_name: 'single-file', version_cmd: 'single-file --version', PATH: '{{BINPROVIDERS.npm.PATH}}'}
-
 
 - name: "Install Singlefile"
   hosts: localhost
@@ -18,12 +10,26 @@
   vars:
     SINGLEFILE_VERSION_EXACT: '1.1.54'
   tasks:
-    - name: Make sure Node and NPM are installed
+    - include_role:
-      assert:
+        name: setup_lib_npm
-        that:
+      vars:
-          - BINARIES.node.version
+        MIN_NODE_VERSION: '20.0.0'
-          - BINARIES.npm.version
+        MIN_NPM_VERSION: '10.0.0'
-        quiet: true
+
+    - name: "Install npm packages: [single-file-cli]"
+      community.general.npm:
+        name: 'single-file-cli@1.1.54'
+        state: "present"
+        path: '{{BINPROVIDERS.npm.root_path}}'
+
+    - name: Load single-file binary from installed NPM package
+      include_role:
+        name: load_binary
+      vars:
+        name: singlefile
+        bin_name: 'single-file'
+        version_cmd: 'single-file --version'
+        PATH: '{{BINPROVIDERS.npm.PATH}}'
 
     - name: Check that installed Singlefile version matches expected version
       assert:

archivebox/playbooks/install_ytdlp.yml

@@ -1,37 +1,24 @@
+#!/usr/bin/env ansible-playbook
 ---
-- import_playbook: install_package.yml
-  vars:
-    install_packages:
-      ffmpeg: {packages: ['ffmpeg']}
-    state: 'present'
-
-- import_playbook: install_pip.yml
-  vars:
-    install_pip:
-      ytdlp: {packages: ['yt-dlp']}
-    state: 'present'
-
-- import_playbook: load_binaries.yml
-  vars:
-    load_binaries:
-      ffmpeg: {bin_name: 'ffmpeg', version_cmd: 'ffmpeg -version'}
-      ytdlp: {bin_name: 'yt-dlp', version_cmd: 'yt-dlp --version'}
-
-
 - name: "Install YT-DLP"
   hosts: localhost
   gather_facts: no
   vars:
     YTDLP_VERSION_MIN: '2024.8.6'
   tasks:
-    - name: Make sure Python and Pip are installed
+    - include_role:
-      assert:
+        name: setup_lib_pip
-        that:
+      vars:
-          - BINARIES.python.version
+        MIN_PYTHON_VERSION: '3.10.0'
-          - BINARIES.pip.version
+        MIN_PIP_VERSION: '22.0'
-          - BINARIES.ffmpeg.version
+    
-          - BINARIES.ytdlp.version
+    - name: "Install pip packages: {{install_pip}}"
-        quiet: true
+      ansible.builtin.pip:
+        name: 'yt-dlp'
+        state: "present"
+        virtualenv: '{{LIB_DIR_PIP}}/venv'
+        virtualenv_python: "{{BINARIES.python.abspath}}"
+        virtualenv_site_packages: yes
 
     - name: Check that installed YT-DLP matches expected version
       assert:
@@ -40,3 +27,23 @@
 
     - debug:
         msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"
+
+
+
+# - import_playbook: install_package.yml
+#   vars:
+#     install_packages:
+#       ffmpeg: {packages: ['ffmpeg']}
+#     state: 'present'
+
+# - import_playbook: install_pip.yml
+#   vars:
+#     install_pip:
+#       ytdlp: {packages: ['yt-dlp']}
+#     state: 'present'
+
+# - import_playbook: load_binaries.yml
+#   vars:
+#     load_binaries:
+#       ffmpeg: {bin_name: 'ffmpeg', version_cmd: 'ffmpeg -version'}
+#       ytdlp: {bin_name: 'yt-dlp', version_cmd: 'yt-dlp --version'}

archivebox/playbooks/load_binaries.yml

@@ -1,3 +1,4 @@
+#!/usr/bin/env ansible-playbook
 ---
 - name: Get binary abspaths and versions and add to BINARIES
   hosts: localhost