mirror of
https://github.com/ArchiveBox/ArchiveBox
synced 2024-11-10 06:34:16 +00:00
revert back to python-3.11 base image and add cache dirs for massive build speed increase
This commit is contained in:
Nick Sweeting
parent
ce4df6d965
commit
562fd5a3bb
3 changed files with 36 additions and 27 deletions
55
Dockerfile
55
Dockerfile
|
|
@ -16,7 +16,7 @@
|
|||
|
||||
|
||||
# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
|
||||
FROM debian:bookworm-backports
|
||||
FROM python:3.11-slim-bookworm
|
||||
|
||||
LABEL name="archivebox" \
|
||||
maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
|
||||
|
|
@ -98,8 +98,9 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
|
|||
# https://docs.linuxserver.io/general/understanding-puid-and-pgid
|
||||
|
||||
# Install system apt dependencies (adding backports to access more recent apt updates)
|
||||
RUN echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
|
||||
# && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \
|
||||
RUN --mount=type=cache,target=/var/cache/apt \
|
||||
echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
|
||||
&& echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \
|
||||
&& mkdir -p /etc/apt/keyrings \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y -t bookworm-backports --no-install-recommends \
|
||||
|
|
@ -114,7 +115,8 @@ RUN echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
|
|||
######### Language Environments ####################################
|
||||
|
||||
# Install Node environment
|
||||
RUN echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
|
||||
RUN --mount=type=cache,target=/var/cache/apt \
|
||||
echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
|
||||
&& echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
|
||||
&& curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
|
||||
&& apt-get update -qq \
|
||||
|
|
@ -131,30 +133,34 @@ RUN echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
|
|||
) | tee -a /VERSION.txt
|
||||
|
||||
# Install Python environment
|
||||
RUN echo "[+] Installing Python $PYTHON_VERSION environment in $GLOBAL_VENV and $APP_VENV..." \
|
||||
RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=/var/cache/apt \
|
||||
echo "[+] Installing Python $PYTHON_VERSION environment in $GLOBAL_VENV and $APP_VENV..." \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y -t bookworm-backports --no-install-recommends \
|
||||
python3 python3-pip python3-setuptools python3-wheel python3-venv python3-pdm python-dev-is-python3 \
|
||||
python3 python3-pip python3-setuptools python3-wheel python3-venv python-dev-is-python3 \
|
||||
#python3-ldap libldap2-dev libsasl2-dev libssl-dev python3-msgpack \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
# tell PDM to allow using global system python site packages
|
||||
&& rm /usr/lib/python3*/EXTERNALLY-MANAGED \
|
||||
# && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
|
||||
# create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
|
||||
&& python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
|
||||
# && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
|
||||
# && python3 -m venv --system-site-packages $GLOBAL_VENV \
|
||||
# && python3 -m venv $GLOBAL_VENV \
|
||||
# install global dependencies / python build dependencies in GLOBAL_VENV
|
||||
&& $GLOBAL_VENV/bin/pip install --break-system-packages --upgrade pip pdm setuptools wheel \
|
||||
&& pip install --upgrade pip setuptools wheel \
|
||||
# Save version info
|
||||
&& ( \
|
||||
which python3 && python3 --version | grep " $PYTHON_VERSION" \
|
||||
&& which pip3 && pip3 --version \
|
||||
&& which pdm && pdm --version \
|
||||
&& which pip && pip --version \
|
||||
# && which pdm && pdm --version \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
######### Extractor Dependencies ##################################
|
||||
|
||||
# Install apt dependencies
|
||||
RUN echo "[+] Installing APT extractor dependencies globally using apt..." \
|
||||
RUN --mount=type=cache,target=/var/cache/apt \
|
||||
echo "[+] Installing APT extractor dependencies globally using apt..." \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y -t bookworm-backports --no-install-recommends \
|
||||
curl wget git yt-dlp ffmpeg ripgrep \
|
||||
|
|
@ -173,11 +179,12 @@ RUN echo "[+] Installing APT extractor dependencies globally using apt..." \
|
|||
) | tee -a /VERSION.txt
|
||||
|
||||
# Install chromium browser using playwright
|
||||
RUN echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
|
||||
RUN --mount=type=cache,target=/var/cache/apt \
|
||||
echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
|
||||
&& apt-get update -qq \
|
||||
&& if [[ "$TARGETPLATFORM" == "linux/amd64" || "$TARGETPLATFORM" == "linux/arm64" ]]; then \
|
||||
# install Chromium using playwright
|
||||
$GLOBAL_VENV/bin/pip install playwright \
|
||||
pip install playwright \
|
||||
&& $GLOBAL_VENV/bin/playwright install --with-deps chromium \
|
||||
&& export CHROME_BINARY="$($GLOBAL_VENV/bin/python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')"; \
|
||||
else \
|
||||
|
|
@ -199,8 +206,9 @@ RUN echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PAT
|
|||
# Install Node dependencies
|
||||
WORKDIR "$CODE_DIR"
|
||||
COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR/"
|
||||
RUN echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
|
||||
&& npm ci --prefer-offline --no-audit \
|
||||
RUN --mount=type=cache,target=/root/.npm \
|
||||
echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
|
||||
&& npm ci --prefer-offline --no-audit --cache /root/.npm \
|
||||
&& ( \
|
||||
which node && node --version \
|
||||
&& which npm && npm version \
|
||||
|
|
@ -211,18 +219,18 @@ RUN echo "[+] Installing NPM extractor dependencies from package.json into $NODE
|
|||
|
||||
# Install ArchiveBox Python dependencies
|
||||
WORKDIR "$CODE_DIR"
|
||||
COPY --chown=root:root --chmod=755 "./pyproject.toml" "./pdm.lock" "requirements.txt" "$CODE_DIR/"
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
&& echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt into $GLOBAL_VENV..." \
|
||||
COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR/"
|
||||
RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=/var/cache/apt \
|
||||
echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt into $GLOBAL_VENV..." \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y -t bookworm-backports --no-install-recommends \
|
||||
build-essential libssl-dev libldap2-dev libsasl2-dev \
|
||||
&& ln -s "$GLOBAL_VENV" "$APP_VENV" \
|
||||
# && ln -s "$GLOBAL_VENV" "$APP_VENV" \
|
||||
# && pdm use --venv in-project \
|
||||
# && pdm run python -m ensurepip \
|
||||
# && pdm sync --fail-fast --no-editable --group :all --no-self \
|
||||
# && pdm export -o requirements.txt --without-hashes \
|
||||
&& source $GLOBAL_VENV/bin/activate \
|
||||
# && source $GLOBAL_VENV/bin/activate \
|
||||
&& pip install -r requirements.txt \
|
||||
&& apt-get purge -y \
|
||||
build-essential libssl-dev libldap2-dev libsasl2-dev \
|
||||
|
|
@ -232,7 +240,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
|
|||
|
||||
# Install ArchiveBox Python package from source
|
||||
COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
|
||||
RUN echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
|
||||
RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=/var/cache/apt \
|
||||
echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
|
||||
&& apt-get update -qq \
|
||||
# install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
|
||||
&& if [[ "$TARGETPLATFORM" == "linux/arm/v7" ]]; then \
|
||||
|
|
@ -243,7 +252,7 @@ RUN echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
|
|||
procps; \
|
||||
fi \
|
||||
# INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
|
||||
&& $GLOBAL_VENV/bin/pip3 install --break-system-packages -e "$CODE_DIR"[sonic,ldap] \
|
||||
&& pip install --break-system-packages -e "$CODE_DIR"[sonic,ldap] \
|
||||
# save docker image size and always remove compilers / build tools after building is complete
|
||||
&& apt-get purge -y build-essential \
|
||||
&& apt-get autoremove -y \
|
||||
|
|
|
|||
6
pdm.lock
6
pdm.lock
|
|
@ -443,7 +443,7 @@ files = [
|
|||
|
||||
[[package]]
|
||||
name = "ipython"
|
||||
version = "8.17.1"
|
||||
version = "8.17.2"
|
||||
requires_python = ">=3.9"
|
||||
summary = "IPython: Productive Interactive Computing"
|
||||
dependencies = [
|
||||
|
|
@ -461,8 +461,8 @@ dependencies = [
|
|||
"typing-extensions; python_version < \"3.10\"",
|
||||
]
|
||||
files = [
|
||||
{file = "ipython-8.17.1-py3-none-any.whl", hash = "sha256:b4510d0a163c89c78cf81be68acc841dde4a8290b6ee3f3c9578ccdd2574553c"},
|
||||
{file = "ipython-8.17.1.tar.gz", hash = "sha256:9e12020f4bf74f08631c0f033bb580e074fbe36c64903195f3e63b9c0a986cbe"},
|
||||
{file = "ipython-8.17.2-py3-none-any.whl", hash = "sha256:1e4d1d666a023e3c93585ba0d8e962867f7a111af322efff6b9c58062b3e5444"},
|
||||
{file = "ipython-8.17.2.tar.gz", hash = "sha256:126bb57e1895594bb0d91ea3090bbd39384f6fe87c3d57fd558d0670f50339bb"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ django-extensions==3.1.5
|
|||
executing==2.0.1
|
||||
greenlet==3.0.0
|
||||
idna==3.4
|
||||
ipython==8.17.1
|
||||
ipython==8.17.2
|
||||
jedi==0.19.1
|
||||
matplotlib-inline==0.1.6
|
||||
mutagen==1.47.0
|
||||
|
|
|
|||
Loading…
Reference in a new issue