Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-14 00:17:08 +00:00
Code Issues Projects Releases Packages Wiki Activity

fix cache locking errors during parallel builds

Browse source
This commit is contained in:
Nick Sweeting 2023-10-31 14:59:17 -07:00
parent 2c692191ef
commit 41bd7c6480
3 changed files with 10 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/docker.yml vendored
View file

@ -33,9 +33,6 @@ jobs:
version: latest version: latest
install: true install: true
platforms: linux/amd64,linux/arm64,linux/arm/v7 platforms: linux/amd64,linux/arm64,linux/arm/v7
config-inline: |
[worker.oci]
max-parallelism = 1
- name: Builder instance name - name: Builder instance name
run: echo ${{ steps.buildx.outputs.name }} run: echo ${{ steps.buildx.outputs.name }}

16
Dockerfile
View file

@ -100,7 +100,7 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
# https://docs.linuxserver.io/general/understanding-puid-and-pgid # https://docs.linuxserver.io/general/understanding-puid-and-pgid
# Install system apt dependencies (adding backports to access more recent apt updates) # Install system apt dependencies (adding backports to access more recent apt updates)
RUN --mount=type=cache,target=/var/cache/apt \ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \ echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
&& echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \ && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \
&& mkdir -p /etc/apt/keyrings \ && mkdir -p /etc/apt/keyrings \
@ -117,7 +117,7 @@ RUN --mount=type=cache,target=/var/cache/apt \
######### Language Environments #################################### ######### Language Environments ####################################
# Install Node environment # Install Node environment
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/root/.npm \ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/root/.npm,sharing=locked \
echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \ echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
&& echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \ && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
&& curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
@ -135,7 +135,7 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/root/.np
) | tee -a /VERSION.txt ) | tee -a /VERSION.txt
# Install Python environment # Install Python environment
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/root/.cache/pip \ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/root/.cache/pip,sharing=locked \
echo "[+] Setting up Python $PYTHON_VERSION runtime..." \ echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
# tell PDM to allow using global system python site packages # tell PDM to allow using global system python site packages
# && rm /usr/lib/python3*/EXTERNALLY-MANAGED \ # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
@ -156,7 +156,7 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/root/.ca
######### Extractor Dependencies ################################## ######### Extractor Dependencies ##################################
# Install apt dependencies # Install apt dependencies
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/root/.cache/pip \ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/root/.cache/pip,sharing=locked \
echo "[+] Installing APT extractor dependencies globally using apt..." \ echo "[+] Installing APT extractor dependencies globally using apt..." \
&& apt-get update -qq \ && apt-get update -qq \
&& apt-get install -qq -y -t bookworm-backports --no-install-recommends \ && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
@ -176,7 +176,7 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/root/.ca
) | tee -a /VERSION.txt ) | tee -a /VERSION.txt
# Install chromium browser using playwright # Install chromium browser using playwright
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=/root/.cache/ms-playwright \ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/root/.cache/pip,sharing=locked --mount=type=cache,target=/root/.cache/ms-playwright \
echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \ echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
&& apt-get update -qq \ && apt-get update -qq \
&& if [[ "$TARGETPLATFORM" == *amd64* || "$TARGETPLATFORM" == *arm64* ]]; then \ && if [[ "$TARGETPLATFORM" == *amd64* || "$TARGETPLATFORM" == *arm64* ]]; then \
@ -204,7 +204,7 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/root/.ca
# Install Node dependencies # Install Node dependencies
WORKDIR "$CODE_DIR" WORKDIR "$CODE_DIR"
COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR/" COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR/"
RUN --mount=type=cache,target=/root/.npm \ RUN --mount=type=cache,target=/root/.npm,sharing=locked \
echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \ echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
&& npm ci --prefer-offline --no-audit --cache /root/.npm \ && npm ci --prefer-offline --no-audit --cache /root/.npm \
&& ( \ && ( \
@ -218,7 +218,7 @@ RUN --mount=type=cache,target=/root/.npm \
# Install ArchiveBox Python dependencies # Install ArchiveBox Python dependencies
WORKDIR "$CODE_DIR" WORKDIR "$CODE_DIR"
COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR/" COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR/"
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/root/.cache/pip \ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/root/.cache/pip,sharing=locked \
echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \ echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
&& apt-get update -qq \ && apt-get update -qq \
&& apt-get install -qq -y -t bookworm-backports --no-install-recommends \ && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
@ -240,7 +240,7 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/root/.ca
# Install ArchiveBox Python package from source # Install ArchiveBox Python package from source
COPY --chown=root:root --chmod=755 "." "$CODE_DIR/" COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/root/.cache/pip \ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/root/.cache/pip,sharing=locked \
echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \ echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
&& apt-get update -qq \ && apt-get update -qq \
# install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi

2
requirements.txt
View file

@ -13,6 +13,7 @@ decorator==5.1.1
django==3.1.14 django==3.1.14
django-auth-ldap==4.1.0 django-auth-ldap==4.1.0
django-extensions==3.1.5 django-extensions==3.1.5
exceptiongroup==1.1.3
executing==2.0.1 executing==2.0.1
idna==3.4 idna==3.4
ipython==8.17.2 ipython==8.17.2
@ -40,6 +41,7 @@ sonic-client==1.0.0
sqlparse==0.4.4 sqlparse==0.4.4
stack-data==0.6.3 stack-data==0.6.3
traitlets==5.13.0 traitlets==5.13.0
typing-extensions==4.8.0
tzlocal==5.2 tzlocal==5.2
urllib3==2.0.7 urllib3==2.0.7
w3lib==2.1.2 w3lib==2.1.2