From faa6b3969d632a899c9c762791b2777abb39cecd Mon Sep 17 00:00:00 2001 From: VibroAxe Date: Tue, 20 Jul 2021 12:04:38 +0000 Subject: [PATCH 1/6] Rebase generic off monolithic This commit rebases the generic code into the monolithic image This allows the base image to be the more regularly accessed / edited image Commits to lancachenet/generic:switcheroo will mimic this commit Also fixes lancachenet/generic#108 --- .circleci/config.yml | 56 ++++++++------- .gitignore | 1 + Dockerfile | 22 +++++- goss.yaml | 2 +- .../etc/nginx/conf.d/20_proxy_cache_path.conf | 1 + overlay/etc/nginx/nginx.conf | 39 +++++++++++ .../etc/nginx/sites-available/10_cache.conf | 10 +++ .../sites-available/cache.conf.d/10_root.conf | 8 +++ .../sites-available/cache.conf.d/20_lol.conf | 5 ++ .../cache.conf.d/21_arenanet_manifest.conf | 6 ++ .../cache.conf.d/22_wsus_cabs.conf | 6 ++ .../cache.conf.d/90_lancache_heartbeat.conf | 6 ++ .../cache.conf.d/root/10_loop_detection.conf | 7 ++ .../cache.conf.d/root/20_cache.conf | 30 ++++++++ .../root/30_cache_key.conf | 0 .../cache.conf.d/root/40_etags.conf | 3 + .../cache.conf.d/root/90_upstream.conf | 10 +++ .../cache.conf.d/root/99_debug_header.conf | 4 ++ .../etc/nginx/stream-available/10_sni.conf | 9 +++ overlay/etc/nginx/workers.conf | 1 + overlay/etc/supervisor/conf.d/heartbeat.conf | 4 ++ .../hooks/entrypoint-pre.d/05_config_check.sh | 46 +++++++++++++ overlay/hooks/entrypoint-pre.d/10_setup.sh | 14 ++++ .../hooks/entrypoint-pre.d/20_perms_check.sh | 14 ++++ .../supervisord-pre.d/99_config_check.sh | 9 +++ overlay/scripts/cache_test.sh | 29 ++++++++ overlay/scripts/getconfig.sh | 39 +++++++++++ overlay/scripts/heartbeat.sh | 16 +++++ run-tests.sh | 69 ++----------------- 29 files changed, 375 insertions(+), 91 deletions(-) create mode 100644 .gitignore create mode 100644 overlay/etc/nginx/conf.d/20_proxy_cache_path.conf create mode 100644 overlay/etc/nginx/nginx.conf create mode 100644 overlay/etc/nginx/sites-available/10_cache.conf create mode 100644 overlay/etc/nginx/sites-available/cache.conf.d/10_root.conf create mode 100644 overlay/etc/nginx/sites-available/cache.conf.d/20_lol.conf create mode 100644 overlay/etc/nginx/sites-available/cache.conf.d/21_arenanet_manifest.conf create mode 100644 overlay/etc/nginx/sites-available/cache.conf.d/22_wsus_cabs.conf create mode 100644 overlay/etc/nginx/sites-available/cache.conf.d/90_lancache_heartbeat.conf create mode 100644 overlay/etc/nginx/sites-available/cache.conf.d/root/10_loop_detection.conf create mode 100644 overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf rename overlay/etc/nginx/sites-available/{generic.conf.d => cache.conf.d}/root/30_cache_key.conf (100%) create mode 100644 overlay/etc/nginx/sites-available/cache.conf.d/root/40_etags.conf create mode 100644 overlay/etc/nginx/sites-available/cache.conf.d/root/90_upstream.conf create mode 100644 overlay/etc/nginx/sites-available/cache.conf.d/root/99_debug_header.conf create mode 100644 overlay/etc/nginx/stream-available/10_sni.conf create mode 100644 overlay/etc/nginx/workers.conf create mode 100644 overlay/etc/supervisor/conf.d/heartbeat.conf create mode 100644 overlay/hooks/entrypoint-pre.d/05_config_check.sh create mode 100644 overlay/hooks/entrypoint-pre.d/10_setup.sh create mode 100644 overlay/hooks/entrypoint-pre.d/20_perms_check.sh create mode 100644 overlay/hooks/supervisord-pre.d/99_config_check.sh create mode 100755 overlay/scripts/cache_test.sh create mode 100755 overlay/scripts/getconfig.sh create mode 100755 overlay/scripts/heartbeat.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 86cbb0f..92c54aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,5 @@ workflows: - version: 2 + version: 2.1 build_test_deploy: jobs: - test @@ -11,22 +11,26 @@ workflows: branches: only: - master + - build_children: + context: circle-api + requires: + - publish_latest -version: 2 +version: 2.1 +executors: + testbuild-executor: + machine: + image: ubuntu-1604:201903-01 jobs: test: - docker: - - image: circleci/python:2-jessie + executor: testbuild-executor steps: - checkout - - - setup_remote_docker: # (2) - docker_layer_caching: false # (3) - run: name: Install goss command: | # rather than give internet scripts SU rights, we install to local user bin and add to path - mkdir ~/bin + [ -d ~/bin ] || mkdir ~/bin export GOSS_DST=~/bin export PATH=$PATH:~/bin curl -fsSL https://goss.rocks/install | sh @@ -36,39 +40,45 @@ jobs: command: | # Don't forget path! export PATH=$PATH:~/bin - # Important, change from mount to work on remote docker, see https://github.com/aelsabbahy/goss/pull/271 - # If using machine image you do not need this. - export GOSS_FILES_STRATEGY=cp - ./run-tests.sh circleci keepimage + ./run-tests.sh --circleci --keepimage - run: name: Save docker image command: | - mkdir -p workspace + [ -d workspace ] || mkdir workspace docker save -o workspace/lancachenet-monolithic.tar lancachenet/monolithic:goss-test - - persist_to_workspace: - root: workspace - paths: - lancachenet-monolithic.tar + #Download from Artifacts and Load this into your own docker using the following command + #docker load -i /tmp/workspace/lancachenet-monolithic.tar - store_test_results: - path: reports + path: reports/goss/report.xml - store_artifacts: path: reports destination: reports - store_artifacts: path: workspace/lancachenet-monolithic.tar destination: docker-lancachenet-monolithic.tar + - persist_to_workspace: + root: workspace + paths: + lancachenet-monolithic.tar publish_latest: - docker: - - image: circleci/python:2-jessie + executor: testbuild-executor steps: - - setup_remote_docker: # (2) - docker_layer_caching: false # (3) - attach_workspace: at: /tmp/workspace - run: name: "Deploy latest to docker hub" command: | - docker login -u $DOCKER_USER -p $DOCKER_PASS docker load -i /tmp/workspace/lancachenet-monolithic.tar + docker login -u $DOCKER_USER -p $DOCKER_PASS docker tag lancachenet/monolithic:goss-test lancachenet/monolithic:latest docker push lancachenet/monolithic:latest + build_children: + executor: testbuild-executor + steps: + - run: + name: "Request API to build children" + command: | + for child in {"monolithic"}; do + echo "Asking API to trigger build for $child" + curl -X POST --header "Content-Type: application/json" -d '{"branch":"master"}' https://circleci.com/api/v1.1/project/github/lancachenet/$child/build?circle-token=${CIRCLE_API_USER_TOKEN} + done diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..27a3afb --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +reports diff --git a/Dockerfile b/Dockerfile index e351260..fe40d75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ -FROM lancachenet/generic:latest -MAINTAINER LanCache.Net Team +FROM lancachenet/ubuntu-nginx:latest +LABEL version=3 +LABEL description="Single caching container for caching game content at lan parties." +LABEL maintainer=" LanCache.Net Team " ENV GENERICCACHE_VERSION=2 \ CACHE_MODE=monolithic \ @@ -17,6 +19,20 @@ ENV GENERICCACHE_VERSION=2 \ COPY overlay/ / +RUN rm /etc/nginx/sites-enabled/* /etc/nginx/stream-enabled/* ;\ + rm /etc/nginx/conf.d/gzip.conf ;\ + chmod 754 /var/log/tallylog ; \ + id -u ${WEBUSER} &> /dev/null || adduser --system --home /var/www/ --no-create-home --shell /bin/false --group --disabled-login ${WEBUSER} ;\ + chmod 755 /scripts/* ;\ + mkdir -m 755 -p /data/cache ;\ + mkdir -m 755 -p /data/info ;\ + mkdir -m 755 -p /data/logs ;\ + mkdir -m 755 -p /tmp/nginx/ ;\ + chown -R ${WEBUSER}:${WEBUSER} /data/ ;\ + mkdir -p /etc/nginx/sites-enabled ;\ + ln -s /etc/nginx/sites-available/10_cache.conf /etc/nginx/sites-enabled/10_generic.conf; \ + ln -s /etc/nginx/stream-available/10_sni.conf /etc/nginx/stream-enabled/10_sni.conf + RUN mkdir -m 755 -p /data/cachedomains ;\ mkdir -m 755 -p /tmp/nginx ;\ apt-get update ;\ @@ -26,5 +42,5 @@ RUN git clone --depth=1 --no-single-branch https://github.com/uklans/cache-domai VOLUME ["/data/logs", "/data/cache", "/data/cachedomains", "/var/www"] -EXPOSE 80 +EXPOSE 80 443 WORKDIR /scripts diff --git a/goss.yaml b/goss.yaml index a62b888..54e5354 100644 --- a/goss.yaml +++ b/goss.yaml @@ -13,7 +13,7 @@ command: exit-status: 0 stdout: - Succesfully Cached - timeout: 10000 + timeout: 20000 process: nginx: running: true diff --git a/overlay/etc/nginx/conf.d/20_proxy_cache_path.conf b/overlay/etc/nginx/conf.d/20_proxy_cache_path.conf new file mode 100644 index 0000000..8b43a8f --- /dev/null +++ b/overlay/etc/nginx/conf.d/20_proxy_cache_path.conf @@ -0,0 +1 @@ +proxy_cache_path /data/cache/cache levels=2:2 keys_zone=generic:CACHE_MEM_SIZE inactive=200d max_size=CACHE_DISK_SIZE loader_files=1000 loader_sleep=50ms loader_threshold=300ms use_temp_path=off; diff --git a/overlay/etc/nginx/nginx.conf b/overlay/etc/nginx/nginx.conf new file mode 100644 index 0000000..367a734 --- /dev/null +++ b/overlay/etc/nginx/nginx.conf @@ -0,0 +1,39 @@ +user www-data; +include /etc/nginx/workers.conf; +pid /run/nginx.pid; + +include /etc/nginx/modules-enabled/*.conf; + +events { + worker_connections 4096; + multi_accept on; + use epoll; +} + +http { + aio threads; + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + + gzip on; + + include /etc/nginx/conf.d/*.conf; + + include /etc/nginx/sites-enabled/*.conf; +} + + +stream { + include /etc/nginx/stream.d/*.conf; + include /etc/nginx/stream-enabled/*; +} diff --git a/overlay/etc/nginx/sites-available/10_cache.conf b/overlay/etc/nginx/sites-available/10_cache.conf new file mode 100644 index 0000000..1976708 --- /dev/null +++ b/overlay/etc/nginx/sites-available/10_cache.conf @@ -0,0 +1,10 @@ + +server { + listen 80 reuseport; + + access_log /data/logs/access.log cachelog; + error_log /data/logs/error.log; + + + include /etc/nginx/sites-available/cache.conf.d/*.conf; +} diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/10_root.conf b/overlay/etc/nginx/sites-available/cache.conf.d/10_root.conf new file mode 100644 index 0000000..b99288e --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/10_root.conf @@ -0,0 +1,8 @@ + resolver UPSTREAM_DNS ipv6=off; + + location / { + + include /etc/nginx/sites-available/cache.conf.d/root/*.conf; + + } + diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/20_lol.conf b/overlay/etc/nginx/sites-available/cache.conf.d/20_lol.conf new file mode 100644 index 0000000..ac58b17 --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/20_lol.conf @@ -0,0 +1,5 @@ + + # Fix for League of Legends Updater + location ~ ^.+(releaselisting_.*|.version$) { + proxy_pass http://$host; + } diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/21_arenanet_manifest.conf b/overlay/etc/nginx/sites-available/cache.conf.d/21_arenanet_manifest.conf new file mode 100644 index 0000000..509d614 --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/21_arenanet_manifest.conf @@ -0,0 +1,6 @@ + # Fix for GW2 manifest + location ^~ /latest64 { + proxy_cache_bypass 1; + proxy_no_cache 1; + proxy_pass http://$host$request_uri; + } diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/22_wsus_cabs.conf b/overlay/etc/nginx/sites-available/cache.conf.d/22_wsus_cabs.conf new file mode 100644 index 0000000..4af9591 --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/22_wsus_cabs.conf @@ -0,0 +1,6 @@ + # Fix for WSUS authroot cab files + location ~* (authrootstl.cab|pinrulesstl.cab|disallowedcertstl.cab)$ { + proxy_cache_bypass 1; + proxy_no_cache 1; + proxy_pass http://$host$request_uri; + } diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/90_lancache_heartbeat.conf b/overlay/etc/nginx/sites-available/cache.conf.d/90_lancache_heartbeat.conf new file mode 100644 index 0000000..deacc43 --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/90_lancache_heartbeat.conf @@ -0,0 +1,6 @@ + location = /lancache-heartbeat { + add_header X-LanCache-Processed-By $hostname; + add_header 'Access-Control-Expose-Headers' '*'; + add_header 'Access-Control-Allow-Origin' '*'; + return 204; + } diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/root/10_loop_detection.conf b/overlay/etc/nginx/sites-available/cache.conf.d/root/10_loop_detection.conf new file mode 100644 index 0000000..067989c --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/root/10_loop_detection.conf @@ -0,0 +1,7 @@ + # Abort any circular requests + if ($http_X_LanCache_Processed_By = $hostname) { + return 508; + } + + proxy_set_header X-LanCache-Processed-By $hostname; + add_header X-LanCache-Processed-By $hostname,$http_X_LanCache_Processed_By; diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf b/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf new file mode 100644 index 0000000..294a6a4 --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf @@ -0,0 +1,30 @@ + # Cache Location + slice 1m; + proxy_cache generic; + + proxy_ignore_headers Expires Cache-Control; + proxy_cache_valid 200 206 CACHE_MAX_AGE; + proxy_set_header Range $slice_range; + + # Only download one copy at a time and use a large timeout so + # this really happens, otherwise we end up wasting bandwith + # getting the file multiple times. + proxy_cache_lock on; + proxy_cache_lock_timeout 1h; + + # Allow the use of state entries + proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504; + + # Allow caching of 200 but not 301 or 302 as our cache key may not include query params + # hence may not be valid for all users + proxy_cache_valid 301 302 0; + + # Enable cache revalidation + proxy_cache_revalidate on; + + # Don't cache requests marked as nocache=1 + proxy_cache_bypass $arg_nocache; + + # 40G max file + proxy_max_temp_file_size 40960m; + diff --git a/overlay/etc/nginx/sites-available/generic.conf.d/root/30_cache_key.conf b/overlay/etc/nginx/sites-available/cache.conf.d/root/30_cache_key.conf similarity index 100% rename from overlay/etc/nginx/sites-available/generic.conf.d/root/30_cache_key.conf rename to overlay/etc/nginx/sites-available/cache.conf.d/root/30_cache_key.conf diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/root/40_etags.conf b/overlay/etc/nginx/sites-available/cache.conf.d/root/40_etags.conf new file mode 100644 index 0000000..a6d85c9 --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/root/40_etags.conf @@ -0,0 +1,3 @@ + # Battle.net Fix + proxy_hide_header ETag; + diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/root/90_upstream.conf b/overlay/etc/nginx/sites-available/cache.conf.d/root/90_upstream.conf new file mode 100644 index 0000000..a42a604 --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/root/90_upstream.conf @@ -0,0 +1,10 @@ + # Upstream Configuration + proxy_next_upstream error timeout http_404; + proxy_pass http://$host$request_uri; + proxy_redirect off; + proxy_ignore_client_abort on; + + # Upstream request headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/root/99_debug_header.conf b/overlay/etc/nginx/sites-available/cache.conf.d/root/99_debug_header.conf new file mode 100644 index 0000000..694746f --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/root/99_debug_header.conf @@ -0,0 +1,4 @@ + # Debug Headers + add_header X-Upstream-Status $upstream_status; + add_header X-Upstream-Response-Time $upstream_response_time; + add_header X-Upstream-Cache-Status $upstream_cache_status; diff --git a/overlay/etc/nginx/stream-available/10_sni.conf b/overlay/etc/nginx/stream-available/10_sni.conf new file mode 100644 index 0000000..aa28f67 --- /dev/null +++ b/overlay/etc/nginx/stream-available/10_sni.conf @@ -0,0 +1,9 @@ +server { + listen 443; + resolver UPSTREAM_DNS ipv6=off; + proxy_pass $ssl_preread_server_name:443; + ssl_preread on; + + access_log /data/logs/stream-access.log stream_basic; + error_log /data/logs/stream-error.log; +} diff --git a/overlay/etc/nginx/workers.conf b/overlay/etc/nginx/workers.conf new file mode 100644 index 0000000..94e152f --- /dev/null +++ b/overlay/etc/nginx/workers.conf @@ -0,0 +1 @@ +worker_processes 16; diff --git a/overlay/etc/supervisor/conf.d/heartbeat.conf b/overlay/etc/supervisor/conf.d/heartbeat.conf new file mode 100644 index 0000000..004cab3 --- /dev/null +++ b/overlay/etc/supervisor/conf.d/heartbeat.conf @@ -0,0 +1,4 @@ +[program:heartbeat] +command=/scripts/heartbeat.sh +stdout_events_enabled=true +stderr_events_enabled=true diff --git a/overlay/hooks/entrypoint-pre.d/05_config_check.sh b/overlay/hooks/entrypoint-pre.d/05_config_check.sh new file mode 100644 index 0000000..26ed761 --- /dev/null +++ b/overlay/hooks/entrypoint-pre.d/05_config_check.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +echo "Checking cache configuration" + + +print_confighash_warning () { + echo "" + echo "ABORTING STARTUP TO AVOID POTENTIALLY INVALIDATING THE CACHE" + echo "" + echo "If you are happy that this cache is valid with the current config changes" + echo "please delete \`//CONFIGHASH\`" + echo "" + echo "See: https://lancache.net/docs/advanced/config-hash/ for more details" + +} + +DETECTED_CACHE_KEY=`grep proxy_cache_key /etc/nginx/sites-available/cache.conf.d/root/30_cache_key.conf | awk '{print $2}'` +NEWHASH="GENERICCACHE_VERSION=${GENERICCACHE_VERSION};CACHE_MODE=${CACHE_MODE};CACHE_SLICE_SIZE=${CACHE_SLICE_SIZE};CACHE_KEY=${DETECTED_CACHE_KEY}" + +if [ -d /data/cache/cache ]; then + echo " Detected existing cache data, checking config hash for consistency" + if [ -f /data/cache/CONFIGHASH ]; then + OLDHASH=`cat /data/cache/CONFIGHASH` + if [ ${OLDHASH} != ${NEWHASH} ]; then + echo "ERROR: Detected CONFIGHASH does not match current CONFIGHASH" + echo " Detected: ${OLDHASH}" + echo " Current: ${NEWHASH}" + print_confighash_warning ${NEWHASH} + exit -1; + else + echo " CONFIGHASH matches current configuration" + fi + else + echo " Could not find CONFIGHASH for existing cachedata" + echo " This is either an upgrade from an older instance of Lancache" + echo " or CONFIGHASH has been deleted intentionally" + echo "" + echo " Creating CONFIGHASH from current live configuration" + echo " Current: ${NEWHASH}" + echo "" + echo " See: https://lancache.net/docs/advanced/config-hash/ for more details" + fi +fi + +mkdir -p /data/cache/cache +echo ${NEWHASH} > /data/cache/CONFIGHASH diff --git a/overlay/hooks/entrypoint-pre.d/10_setup.sh b/overlay/hooks/entrypoint-pre.d/10_setup.sh new file mode 100644 index 0000000..4b112e5 --- /dev/null +++ b/overlay/hooks/entrypoint-pre.d/10_setup.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -e + +# Preprocess UPSTREAM_DNS to allow for multiple resolvers using the same syntax as lancache-dns +UPSTREAM_DNS="$(echo -n "${UPSTREAM_DNS}" | sed 's/[;]/ /g')" + +echo "worker_processes ${NGINX_WORKER_PROCESSES};" > /etc/nginx/workers.conf +sed -i "s/^user .*/user ${WEBUSER};/" /etc/nginx/nginx.conf +sed -i "s/CACHE_MEM_SIZE/${CACHE_MEM_SIZE}/" /etc/nginx/conf.d/20_proxy_cache_path.conf +sed -i "s/CACHE_DISK_SIZE/${CACHE_DISK_SIZE}/" /etc/nginx/conf.d/20_proxy_cache_path.conf +sed -i "s/CACHE_MAX_AGE/${CACHE_MAX_AGE}/" /etc/nginx/sites-available/cache.conf.d/root/20_cache.conf +sed -i "s/slice 1m;/slice ${CACHE_SLICE_SIZE};/" /etc/nginx/sites-available/cache.conf.d/root/20_cache.conf +sed -i "s/UPSTREAM_DNS/${UPSTREAM_DNS}/" /etc/nginx/sites-available/cache.conf.d/10_root.conf +sed -i "s/UPSTREAM_DNS/${UPSTREAM_DNS}/" /etc/nginx/stream-available/10_sni.conf diff --git a/overlay/hooks/entrypoint-pre.d/20_perms_check.sh b/overlay/hooks/entrypoint-pre.d/20_perms_check.sh new file mode 100644 index 0000000..b686cde --- /dev/null +++ b/overlay/hooks/entrypoint-pre.d/20_perms_check.sh @@ -0,0 +1,14 @@ +#!/bin/bash +if [ -d "/data/cache/cache" ]; then + echo "Running fast permissions check" + ls -l /data/cache/cache | tail --lines=+2 | grep -v ${WEBUSER} > /dev/null + + if [[ $? -eq 0 || "$FORCE_PERMS_CHECK" == "true" ]]; then + echo "Doing full checking of permissions (This WILL take a long time on large caches)..." + find /data \! -user ${WEBUSER} -exec chown ${WEBUSER}:${WEBUSER} '{}' + + echo "Permissions ok" + else + echo "Fast permissions check successful, if you have any permissions error try running with -e FORCE_PERMS_CHECK = true" + fi + +fi diff --git a/overlay/hooks/supervisord-pre.d/99_config_check.sh b/overlay/hooks/supervisord-pre.d/99_config_check.sh new file mode 100644 index 0000000..b83bd7d --- /dev/null +++ b/overlay/hooks/supervisord-pre.d/99_config_check.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -e +echo "Currently configured config:" +/scripts/getconfig.sh /etc/nginx/nginx.conf + +echo "Checking nginx config" +/usr/sbin/nginx -t + + [ $? -ne 0 ] || echo "Config check successful" diff --git a/overlay/scripts/cache_test.sh b/overlay/scripts/cache_test.sh new file mode 100755 index 0000000..0511b1d --- /dev/null +++ b/overlay/scripts/cache_test.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e +pageload1=`curl http://www.worldtimeapi.org/api/timezone/ETC/GMT --resolve www.worldtimeapi.org:80:127.0.0.1` +sleep 5 +pageload2=`curl http://www.worldtimeapi.org/api/timezone/ETC/GMT --resolve www.worldtimeapi.org:80:127.0.0.1` +sleep 5 +pageload3=`curl http://worldtimeapi.org/api/timezone/ETC/GMT --resolve worldtimeapi.org:80:127.0.0.1` +sleep 5 +pageload4=`curl http://worldtimeapi.org/api/timezone/ETC/GMT --resolve worldtimeapi.org:80:127.0.0.1` +if [ "$pageload1" == "$pageload2" ]; then + if [ "$pageload3" == "$pageload4" ]; then + if [ "$pageload1" == "$pageload4" ]; then + #In monolithic pages 1+3 should be different as there is no map for this test case + echo "Error caching test page, pages 1+3 are identical" + exit -3 + else + echo "Succesfully Cached" + exit 0 + fi + + else + echo "Error caching test page, pages 3+4 differed" + exit -2 + fi + +else + echo "Error caching test page, pages 1+2 differed" + exit -1 +fi diff --git a/overlay/scripts/getconfig.sh b/overlay/scripts/getconfig.sh new file mode 100755 index 0000000..21c94f7 --- /dev/null +++ b/overlay/scripts/getconfig.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +get_file_contents() { + FILES=$1 + local FILE; + for FILE in $FILES; do + echo "# Including $FILE" + local LINE + while read LINE; do + CLEANLINE=`echo $LINE | sed -e 's/^[[:space:]]*//g' -e 's/[[:space:]]*\$//g'` + if [[ "$CLEANLINE" =~ ^include ]]; then + local CL_LEN + local INCUDE + CL_LEN=${#CLEANLINE}-9; + INCLUDE=${CLEANLINE:8:$CL_LEN} + get_file_contents "$INCLUDE" + else + echo $LINE + fi + done < $FILE + echo "# Finished including $FILE" + + done +} + + +main() { + + echo "NGINX CONFIG DUMP FOR $1" + + cd `dirname $1` + + get_file_contents $1 + +} + + + +main `readlink -f $1` diff --git a/overlay/scripts/heartbeat.sh b/overlay/scripts/heartbeat.sh new file mode 100755 index 0000000..30ecc7c --- /dev/null +++ b/overlay/scripts/heartbeat.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e +if [[ "$1" == "" ]]; then + BEATTIME=${BEAT_TIME} +else + BEATTIME=$1 + if [[ "$1" == 0 ]]; then + exit 0; + fi +fi + + +while [ 1 ]; do + sleep $BEATTIME; + wget http://127.0.0.1/lancache-heartbeat -S -O - > /dev/null 2>&1 /dev/null +done diff --git a/run-tests.sh b/run-tests.sh index 272e9dd..c2d7267 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -1,68 +1,9 @@ #!/bin/bash -which goss -if [ $? -ne 0 ]; then - echo "Please install goss from https://goss.rocks/install" - echo "For a quick auto install run the following" - echo "curl -fsSL https://goss.rocks/install | sh" - exit $? +if [[ "$@" == *" -- "* ]]; then + SD_LOGLEVEL="-e SUPERVISORD_LOGLEVEL=INFO" +else + SD_LOGLEVEL="-- -e SUPERVISORD_LOGLEVEL=INFO" fi -GOSS_WAIT_OPS="-r 60s -s 1s" - -docker build --tag lancachenet/monolithic:goss-test . -case $1 in - circleci) - shift; - mkdir -p ./reports/goss - if [[ "$1" == "keepimage" ]]; then - KEEPIMAGE=true - shift - fi - export GOSS_OPTS="$GOSS_OPTS --format junit" - export CONTAINER_LOG_OUTPUT="reports/goss/docker.log" - dgoss run $@ lancachenet/monolithic:goss-test > reports/goss/report.xml - #store result for exit code - RESULT=$? - #Ensure non blank docker.log - echo \ -"Container Output: -$(cat reports/goss/docker.log)" \ - > reports/goss/docker.log - #delete the junk that goss currently outputs :( - sed -i '0,/^.*<\/system-err>/d' reports/goss/report.xml - ;; - docker) - shift; - if [[ "$1" == "keepimage" ]]; then - KEEPIMAGE=true - shift - fi - docker run --name monolithic-goss-test $@ lancachenet/monolithic:goss-test - docker stop monolithic-goss-test - docker rm monolithic-goss-test - RESULT=$? - ;; - edit) - shift; - if [[ "$1" == "keepimage" ]]; then - KEEPIMAGE=true - shift - fi - dgoss edit $@ lancachenet/monolithic:goss-test - RESULT=$? - ;; - *) - if [[ "$1" == "keepimage" ]]; then - KEEPIMAGE=true - shift - fi - dgoss run $@ lancachenet/monolithic:goss-test - RESULT=$? - ;; -esac -[[ "$KEEPIMAGE" == "true" ]] || docker rmi lancachenet/monolithic:goss-test - -exit $RESULT +curl -fsSL https://raw.githubusercontent.com/lancachenet/test-suite/master/dgoss-tests.sh | bash -s -- --imagename="lancachenet/monolithic:goss-test" $@ $SD_LOGLEVEL From d77cfdc601d625bac4809088e64e82994788cdce Mon Sep 17 00:00:00 2001 From: VibroAxe Date: Tue, 20 Jul 2021 13:35:05 +0000 Subject: [PATCH 2/6] Exclude steam server status from caching Implements lancachenet/generic#93 as this is no longer a valid comit after switcheroo --- .../sites-available/cache.conf.d/23_steam_server_status.conf | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 overlay/etc/nginx/sites-available/cache.conf.d/23_steam_server_status.conf diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/23_steam_server_status.conf b/overlay/etc/nginx/sites-available/cache.conf.d/23_steam_server_status.conf new file mode 100644 index 0000000..8ed00d3 --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/23_steam_server_status.conf @@ -0,0 +1,4 @@ +location = /server-status { + proxy_no_cache 1; + proxy_cache_bypass 1; +} \ No newline at end of file From 425a75c7da1de29706692240d9a5ebcfa3215821 Mon Sep 17 00:00:00 2001 From: James Kinsman Date: Sat, 9 Oct 2021 00:11:15 +0100 Subject: [PATCH 3/6] Apply suggestions from code review Thanks @gotenxiao Co-authored-by: GotenXiao --- Dockerfile | 4 ++-- overlay/etc/nginx/sites-available/10_cache.conf | 2 -- .../nginx/sites-available/cache.conf.d/10_root.conf | 1 - .../nginx/sites-available/cache.conf.d/20_lol.conf | 1 - .../cache.conf.d/21_arenanet_manifest.conf | 6 +++--- .../sites-available/cache.conf.d/22_wsus_cabs.conf | 8 ++++---- .../cache.conf.d/23_steam_server_status.conf | 8 ++++---- .../cache.conf.d/root/10_loop_detection.conf | 12 ++++++------ overlay/etc/nginx/stream-available/10_sni.conf | 2 +- 9 files changed, 20 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index fe40d75..be99eee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM lancachenet/ubuntu-nginx:latest LABEL version=3 -LABEL description="Single caching container for caching game content at lan parties." -LABEL maintainer=" LanCache.Net Team " +LABEL description="Single caching container for caching game content at LAN parties." +LABEL maintainer="LanCache.Net Team " ENV GENERICCACHE_VERSION=2 \ CACHE_MODE=monolithic \ diff --git a/overlay/etc/nginx/sites-available/10_cache.conf b/overlay/etc/nginx/sites-available/10_cache.conf index 1976708..950f346 100644 --- a/overlay/etc/nginx/sites-available/10_cache.conf +++ b/overlay/etc/nginx/sites-available/10_cache.conf @@ -1,10 +1,8 @@ - server { listen 80 reuseport; access_log /data/logs/access.log cachelog; error_log /data/logs/error.log; - include /etc/nginx/sites-available/cache.conf.d/*.conf; } diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/10_root.conf b/overlay/etc/nginx/sites-available/cache.conf.d/10_root.conf index b99288e..f065502 100644 --- a/overlay/etc/nginx/sites-available/cache.conf.d/10_root.conf +++ b/overlay/etc/nginx/sites-available/cache.conf.d/10_root.conf @@ -5,4 +5,3 @@ include /etc/nginx/sites-available/cache.conf.d/root/*.conf; } - diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/20_lol.conf b/overlay/etc/nginx/sites-available/cache.conf.d/20_lol.conf index ac58b17..5f1e6ed 100644 --- a/overlay/etc/nginx/sites-available/cache.conf.d/20_lol.conf +++ b/overlay/etc/nginx/sites-available/cache.conf.d/20_lol.conf @@ -1,4 +1,3 @@ - # Fix for League of Legends Updater location ~ ^.+(releaselisting_.*|.version$) { proxy_pass http://$host; diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/21_arenanet_manifest.conf b/overlay/etc/nginx/sites-available/cache.conf.d/21_arenanet_manifest.conf index 509d614..0169230 100644 --- a/overlay/etc/nginx/sites-available/cache.conf.d/21_arenanet_manifest.conf +++ b/overlay/etc/nginx/sites-available/cache.conf.d/21_arenanet_manifest.conf @@ -1,6 +1,6 @@ # Fix for GW2 manifest location ^~ /latest64 { - proxy_cache_bypass 1; - proxy_no_cache 1; - proxy_pass http://$host$request_uri; + proxy_cache_bypass 1; + proxy_no_cache 1; + proxy_pass http://$host$request_uri; } diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/22_wsus_cabs.conf b/overlay/etc/nginx/sites-available/cache.conf.d/22_wsus_cabs.conf index 4af9591..9fe085d 100644 --- a/overlay/etc/nginx/sites-available/cache.conf.d/22_wsus_cabs.conf +++ b/overlay/etc/nginx/sites-available/cache.conf.d/22_wsus_cabs.conf @@ -1,6 +1,6 @@ - # Fix for WSUS authroot cab files + # Fix for WSUS authroot cab files location ~* (authrootstl.cab|pinrulesstl.cab|disallowedcertstl.cab)$ { - proxy_cache_bypass 1; - proxy_no_cache 1; - proxy_pass http://$host$request_uri; + proxy_cache_bypass 1; + proxy_no_cache 1; + proxy_pass http://$host$request_uri; } diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/23_steam_server_status.conf b/overlay/etc/nginx/sites-available/cache.conf.d/23_steam_server_status.conf index 8ed00d3..6f4f8df 100644 --- a/overlay/etc/nginx/sites-available/cache.conf.d/23_steam_server_status.conf +++ b/overlay/etc/nginx/sites-available/cache.conf.d/23_steam_server_status.conf @@ -1,4 +1,4 @@ -location = /server-status { - proxy_no_cache 1; - proxy_cache_bypass 1; -} \ No newline at end of file + location = /server-status { + proxy_no_cache 1; + proxy_cache_bypass 1; + } \ No newline at end of file diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/root/10_loop_detection.conf b/overlay/etc/nginx/sites-available/cache.conf.d/root/10_loop_detection.conf index 067989c..478ad36 100644 --- a/overlay/etc/nginx/sites-available/cache.conf.d/root/10_loop_detection.conf +++ b/overlay/etc/nginx/sites-available/cache.conf.d/root/10_loop_detection.conf @@ -1,7 +1,7 @@ - # Abort any circular requests - if ($http_X_LanCache_Processed_By = $hostname) { - return 508; - } + # Abort any circular requests + if ($http_X_LanCache_Processed_By = $hostname) { + return 508; + } - proxy_set_header X-LanCache-Processed-By $hostname; - add_header X-LanCache-Processed-By $hostname,$http_X_LanCache_Processed_By; + proxy_set_header X-LanCache-Processed-By $hostname; + add_header X-LanCache-Processed-By $hostname,$http_X_LanCache_Processed_By; diff --git a/overlay/etc/nginx/stream-available/10_sni.conf b/overlay/etc/nginx/stream-available/10_sni.conf index aa28f67..40161a3 100644 --- a/overlay/etc/nginx/stream-available/10_sni.conf +++ b/overlay/etc/nginx/stream-available/10_sni.conf @@ -1,6 +1,6 @@ server { listen 443; - resolver UPSTREAM_DNS ipv6=off; + resolver UPSTREAM_DNS ipv6=off; proxy_pass $ssl_preread_server_name:443; ssl_preread on; From 2fa012f66742ccd129467722c113857941eb580e Mon Sep 17 00:00:00 2001 From: VibroAxe Date: Sat, 9 Oct 2021 10:24:38 +0000 Subject: [PATCH 4/6] Ported CACHE_INDEX_SIZE PR from generic Ported PR https://github.com/lancachenet/generic/pull/103/ from @crabbey Changes CACHE_MEM_SIZE to CACHE_INDEX_SIZE but maintains backwards compatiblity through deprecation.sh --- Dockerfile | 2 +- overlay/etc/nginx/conf.d/20_proxy_cache_path.conf | 2 +- overlay/hooks/entrypoint-pre.d/00_deprecation.sh | 6 ++++++ overlay/hooks/entrypoint-pre.d/10_setup.sh | 7 ++++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index be99eee..f12f3a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ LABEL maintainer="LanCache.Net Team " ENV GENERICCACHE_VERSION=2 \ CACHE_MODE=monolithic \ WEBUSER=www-data \ - CACHE_MEM_SIZE=500m \ + CACHE_INDEX_SIZE=500m \ CACHE_DISK_SIZE=1000000m \ CACHE_MAX_AGE=3560d \ CACHE_SLICE_SIZE=1m \ diff --git a/overlay/etc/nginx/conf.d/20_proxy_cache_path.conf b/overlay/etc/nginx/conf.d/20_proxy_cache_path.conf index 8b43a8f..d66bdfa 100644 --- a/overlay/etc/nginx/conf.d/20_proxy_cache_path.conf +++ b/overlay/etc/nginx/conf.d/20_proxy_cache_path.conf @@ -1 +1 @@ -proxy_cache_path /data/cache/cache levels=2:2 keys_zone=generic:CACHE_MEM_SIZE inactive=200d max_size=CACHE_DISK_SIZE loader_files=1000 loader_sleep=50ms loader_threshold=300ms use_temp_path=off; +proxy_cache_path /data/cache/cache levels=2:2 keys_zone=generic:CACHE_INDEX_SIZE inactive=200d max_size=CACHE_DISK_SIZE loader_files=1000 loader_sleep=50ms loader_threshold=300ms use_temp_path=off; diff --git a/overlay/hooks/entrypoint-pre.d/00_deprecation.sh b/overlay/hooks/entrypoint-pre.d/00_deprecation.sh index 7ec1347..0587cfa 100644 --- a/overlay/hooks/entrypoint-pre.d/00_deprecation.sh +++ b/overlay/hooks/entrypoint-pre.d/00_deprecation.sh @@ -7,3 +7,9 @@ if [[ ! -z "${CACHE_DOMAIN_REPO}" ]]; then fi +if [[ ! -z "${CACHE_INDEX_SIZE}" ]]; then + echo " *** CACHE_MEM_SIZE has been deprecated in place of CACHE_INDEX_SIZE" + echo " *** Using CACHE_MEM_SIZE as the value" + echo " *** Please update your configuration at your earliest convenience" + CACHE_INDEX_SIZE=$CACHE_MEM_SIZE +fi \ No newline at end of file diff --git a/overlay/hooks/entrypoint-pre.d/10_setup.sh b/overlay/hooks/entrypoint-pre.d/10_setup.sh index 4b112e5..404ca6a 100644 --- a/overlay/hooks/entrypoint-pre.d/10_setup.sh +++ b/overlay/hooks/entrypoint-pre.d/10_setup.sh @@ -1,12 +1,17 @@ #!/bin/sh set -e +# Handle CACHE_MEM_SIZE deprecation +if [[ ! -z "${CACHE_MEM_SIZE}" ]]; then + CACHE_INDEX_SIZE=${CACHE_MEM_SIZE} +fi + # Preprocess UPSTREAM_DNS to allow for multiple resolvers using the same syntax as lancache-dns UPSTREAM_DNS="$(echo -n "${UPSTREAM_DNS}" | sed 's/[;]/ /g')" echo "worker_processes ${NGINX_WORKER_PROCESSES};" > /etc/nginx/workers.conf sed -i "s/^user .*/user ${WEBUSER};/" /etc/nginx/nginx.conf -sed -i "s/CACHE_MEM_SIZE/${CACHE_MEM_SIZE}/" /etc/nginx/conf.d/20_proxy_cache_path.conf +sed -i "s/CACHE_INDEX_SIZE/${CACHE_INDEX_SIZE}/" /etc/nginx/conf.d/20_proxy_cache_path.conf sed -i "s/CACHE_DISK_SIZE/${CACHE_DISK_SIZE}/" /etc/nginx/conf.d/20_proxy_cache_path.conf sed -i "s/CACHE_MAX_AGE/${CACHE_MAX_AGE}/" /etc/nginx/sites-available/cache.conf.d/root/20_cache.conf sed -i "s/slice 1m;/slice ${CACHE_SLICE_SIZE};/" /etc/nginx/sites-available/cache.conf.d/root/20_cache.conf From 51753881f9b7bbe31b37e068c978b7e82498c096 Mon Sep 17 00:00:00 2001 From: VibroAxe Date: Sat, 9 Oct 2021 11:07:12 +0000 Subject: [PATCH 5/6] Switch to using proxy_cache_lock_age This allows us to cache the response in the event a file hasn't been downloaded yet Set to 2m as a really long time to download a single 1m slice from upstream Based on thoughts from https://github.com/lancachenet/generic/pull/113 --- .../nginx/sites-available/cache.conf.d/root/20_cache.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf b/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf index 294a6a4..263b874 100644 --- a/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf +++ b/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf @@ -10,6 +10,10 @@ # this really happens, otherwise we end up wasting bandwith # getting the file multiple times. proxy_cache_lock on; + # If it's taken over a minute to download a 1m file, we are probably stuck! + # Allow the next request to cache + proxy_cache_lock_age 2m; + # If it's totally broken after an hour, stick it in bypass (this shouldn't ever trigger) proxy_cache_lock_timeout 1h; # Allow the use of state entries @@ -28,3 +32,4 @@ # 40G max file proxy_max_temp_file_size 40960m; + From 3f5d8c53ce39e7b9e8c66cf56609d008dce21ff7 Mon Sep 17 00:00:00 2001 From: VibroAxe Date: Sat, 9 Oct 2021 11:56:40 +0000 Subject: [PATCH 6/6] Update build hierarchy Build generic from monolithic --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 92c54aa..f616bd3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,7 +78,7 @@ jobs: - run: name: "Request API to build children" command: | - for child in {"monolithic"}; do + for child in {"generic"}; do echo "Asking API to trigger build for $child" curl -X POST --header "Content-Type: application/json" -d '{"branch":"master"}' https://circleci.com/api/v1.1/project/github/lancachenet/$child/build?circle-token=${CIRCLE_API_USER_TOKEN} done