rename vendor dir to pkgs

.github/workflows/test.yml

@@ -102,7 +102,7 @@ jobs:
         # TODO: remove this exception for windows once we get tests passing on that platform
         if: ${{ !contains(matrix.os, 'windows') }}
         run: |
-          python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist
+          python -m pytest -s --basetemp=tests/out --ignore=archivebox/pkgs
 
   docker_tests:
     runs-on: ubuntu-latest

archivebox/.flake8

@@ -3,4 +3,4 @@ ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E
 select = F,E9,W
 max-line-length = 130
 max-complexity = 10
-exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv
+exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv,data,data*

archivebox/__init__.py

@@ -47,8 +47,8 @@ from .monkey_patches import *                    # noqa
 
 
 # print('LOADING VENDORED LIBRARIES')
-from .vendor import load_vendored_libs           # noqa
+from .pkgs import load_vendored_pkgs             # noqa
-load_vendored_libs()
+load_vendored_pkgs()
 # print('DONE LOADING VENDORED LIBRARIES')
 
 # Load ABX Plugin Specifications + Default Implementations

archivebox/pkgs/__init__.py

@@ -0,0 +1,39 @@
+import sys
+import importlib
+from pathlib import Path
+
+PKGS_DIR = Path(__file__).parent
+
+VENDORED_PKGS = [
+    'abx',
+    # 'pydantic-pkgr',
+]
+
+# scan ./pkgs and add all dirs present to list of available VENDORED_PKGS
+for subdir in reversed(sorted(PKGS_DIR.iterdir())):
+    if subdir.is_dir() and subdir.name not in VENDORED_PKGS and not subdir.name.startswith('_'):
+        VENDORED_PKGS.append(subdir.name)
+
+
+def load_vendored_pkgs():
+    """Add archivebox/vendor to sys.path and import all vendored libraries present within"""
+    if str(PKGS_DIR) not in sys.path:
+        sys.path.append(str(PKGS_DIR))
+    
+    for pkg_name in VENDORED_PKGS:
+        pkg_dir = PKGS_DIR / pkg_name
+        assert pkg_dir.is_dir(), f'Required vendored pkg {pkg_name} could not be found in {pkg_dir}'
+
+        try:
+            lib = importlib.import_module(pkg_name)
+            # print(f"Successfully imported lib from environment {pkg_name}")
+        except ImportError:
+            sys.path.append(str(pkg_dir))
+            try:
+                lib = importlib.import_module(pkg_name)
+                # print(f"Successfully imported lib from vendored fallback {pkg_name}: {inspect.getfile(lib)}")
+            except ImportError as e:
+                print(f"Failed to import lib from environment or vendored fallback {pkg_name}: {e}", file=sys.stderr)
+                sys.exit(1)
+        
+