Merge pull request #133 from lancachenet/switcheroo

Rebasing generic off monolithic

Includes 
https://github.com/lancachenet/generic/pull/103
https://github.com/lancachenet/generic/pull/93
https://github.com/lancachenet/generic/pull/113
https://github.com/lancachenet/generic/issues/108

Thanks to @crabbey @zerotacg @csm10495 and @Candunc
This commit is contained in:
James Kinsman 2021-10-09 13:01:39 +01:00 committed by GitHub
commit 72aef31a82
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 392 additions and 92 deletions

View file

@ -1,5 +1,5 @@
workflows: workflows:
version: 2 version: 2.1
build_test_deploy: build_test_deploy:
jobs: jobs:
- test - test
@ -11,22 +11,26 @@ workflows:
branches: branches:
only: only:
- master - master
- build_children:
context: circle-api
requires:
- publish_latest
version: 2 version: 2.1
executors:
testbuild-executor:
machine:
image: ubuntu-1604:201903-01
jobs: jobs:
test: test:
docker: executor: testbuild-executor
- image: circleci/python:2-jessie
steps: steps:
- checkout - checkout
- setup_remote_docker: # (2)
docker_layer_caching: false # (3)
- run: - run:
name: Install goss name: Install goss
command: | command: |
# rather than give internet scripts SU rights, we install to local user bin and add to path # rather than give internet scripts SU rights, we install to local user bin and add to path
mkdir ~/bin [ -d ~/bin ] || mkdir ~/bin
export GOSS_DST=~/bin export GOSS_DST=~/bin
export PATH=$PATH:~/bin export PATH=$PATH:~/bin
curl -fsSL https://goss.rocks/install | sh curl -fsSL https://goss.rocks/install | sh
@ -36,39 +40,45 @@ jobs:
command: | command: |
# Don't forget path! # Don't forget path!
export PATH=$PATH:~/bin export PATH=$PATH:~/bin
# Important, change from mount to work on remote docker, see https://github.com/aelsabbahy/goss/pull/271 ./run-tests.sh --circleci --keepimage
# If using machine image you do not need this.
export GOSS_FILES_STRATEGY=cp
./run-tests.sh circleci keepimage
- run: - run:
name: Save docker image name: Save docker image
command: | command: |
mkdir -p workspace [ -d workspace ] || mkdir workspace
docker save -o workspace/lancachenet-monolithic.tar lancachenet/monolithic:goss-test docker save -o workspace/lancachenet-monolithic.tar lancachenet/monolithic:goss-test
- persist_to_workspace: #Download from Artifacts and Load this into your own docker using the following command
root: workspace #docker load -i /tmp/workspace/lancachenet-monolithic.tar
paths:
lancachenet-monolithic.tar
- store_test_results: - store_test_results:
path: reports path: reports/goss/report.xml
- store_artifacts: - store_artifacts:
path: reports path: reports
destination: reports destination: reports
- store_artifacts: - store_artifacts:
path: workspace/lancachenet-monolithic.tar path: workspace/lancachenet-monolithic.tar
destination: docker-lancachenet-monolithic.tar destination: docker-lancachenet-monolithic.tar
- persist_to_workspace:
root: workspace
paths:
lancachenet-monolithic.tar
publish_latest: publish_latest:
docker: executor: testbuild-executor
- image: circleci/python:2-jessie
steps: steps:
- setup_remote_docker: # (2)
docker_layer_caching: false # (3)
- attach_workspace: - attach_workspace:
at: /tmp/workspace at: /tmp/workspace
- run: - run:
name: "Deploy latest to docker hub" name: "Deploy latest to docker hub"
command: | command: |
docker login -u $DOCKER_USER -p $DOCKER_PASS
docker load -i /tmp/workspace/lancachenet-monolithic.tar docker load -i /tmp/workspace/lancachenet-monolithic.tar
docker login -u $DOCKER_USER -p $DOCKER_PASS
docker tag lancachenet/monolithic:goss-test lancachenet/monolithic:latest docker tag lancachenet/monolithic:goss-test lancachenet/monolithic:latest
docker push lancachenet/monolithic:latest docker push lancachenet/monolithic:latest
build_children:
executor: testbuild-executor
steps:
- run:
name: "Request API to build children"
command: |
for child in {"generic"}; do
echo "Asking API to trigger build for $child"
curl -X POST --header "Content-Type: application/json" -d '{"branch":"master"}' https://circleci.com/api/v1.1/project/github/lancachenet/$child/build?circle-token=${CIRCLE_API_USER_TOKEN}
done

1
.gitignore vendored Normal file
View file

@ -0,0 +1 @@
reports

View file

@ -1,10 +1,12 @@
FROM lancachenet/generic:latest FROM lancachenet/ubuntu-nginx:latest
MAINTAINER LanCache.Net Team <team@lancache.net> LABEL version=3
LABEL description="Single caching container for caching game content at LAN parties."
LABEL maintainer="LanCache.Net Team <team@lancache.net>"
ENV GENERICCACHE_VERSION=2 \ ENV GENERICCACHE_VERSION=2 \
CACHE_MODE=monolithic \ CACHE_MODE=monolithic \
WEBUSER=www-data \ WEBUSER=www-data \
CACHE_MEM_SIZE=500m \ CACHE_INDEX_SIZE=500m \
CACHE_DISK_SIZE=1000000m \ CACHE_DISK_SIZE=1000000m \
CACHE_MAX_AGE=3560d \ CACHE_MAX_AGE=3560d \
CACHE_SLICE_SIZE=1m \ CACHE_SLICE_SIZE=1m \
@ -17,6 +19,20 @@ ENV GENERICCACHE_VERSION=2 \
COPY overlay/ / COPY overlay/ /
RUN rm /etc/nginx/sites-enabled/* /etc/nginx/stream-enabled/* ;\
rm /etc/nginx/conf.d/gzip.conf ;\
chmod 754 /var/log/tallylog ; \
id -u ${WEBUSER} &> /dev/null || adduser --system --home /var/www/ --no-create-home --shell /bin/false --group --disabled-login ${WEBUSER} ;\
chmod 755 /scripts/* ;\
mkdir -m 755 -p /data/cache ;\
mkdir -m 755 -p /data/info ;\
mkdir -m 755 -p /data/logs ;\
mkdir -m 755 -p /tmp/nginx/ ;\
chown -R ${WEBUSER}:${WEBUSER} /data/ ;\
mkdir -p /etc/nginx/sites-enabled ;\
ln -s /etc/nginx/sites-available/10_cache.conf /etc/nginx/sites-enabled/10_generic.conf; \
ln -s /etc/nginx/stream-available/10_sni.conf /etc/nginx/stream-enabled/10_sni.conf
RUN mkdir -m 755 -p /data/cachedomains ;\ RUN mkdir -m 755 -p /data/cachedomains ;\
mkdir -m 755 -p /tmp/nginx ;\ mkdir -m 755 -p /tmp/nginx ;\
apt-get update ;\ apt-get update ;\
@ -26,5 +42,5 @@ RUN git clone --depth=1 --no-single-branch https://github.com/uklans/cache-domai
VOLUME ["/data/logs", "/data/cache", "/data/cachedomains", "/var/www"] VOLUME ["/data/logs", "/data/cache", "/data/cachedomains", "/var/www"]
EXPOSE 80 EXPOSE 80 443
WORKDIR /scripts WORKDIR /scripts

View file

@ -13,7 +13,7 @@ command:
exit-status: 0 exit-status: 0
stdout: stdout:
- Succesfully Cached - Succesfully Cached
timeout: 10000 timeout: 20000
process: process:
nginx: nginx:
running: true running: true

View file

@ -0,0 +1 @@
proxy_cache_path /data/cache/cache levels=2:2 keys_zone=generic:CACHE_INDEX_SIZE inactive=200d max_size=CACHE_DISK_SIZE loader_files=1000 loader_sleep=50ms loader_threshold=300ms use_temp_path=off;

View file

@ -0,0 +1,39 @@
user www-data;
include /etc/nginx/workers.conf;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
events {
worker_connections 4096;
multi_accept on;
use epoll;
}
http {
aio threads;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
gzip on;
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*.conf;
}
stream {
include /etc/nginx/stream.d/*.conf;
include /etc/nginx/stream-enabled/*;
}

View file

@ -0,0 +1,8 @@
server {
listen 80 reuseport;
access_log /data/logs/access.log cachelog;
error_log /data/logs/error.log;
include /etc/nginx/sites-available/cache.conf.d/*.conf;
}

View file

@ -0,0 +1,7 @@
resolver UPSTREAM_DNS ipv6=off;
location / {
include /etc/nginx/sites-available/cache.conf.d/root/*.conf;
}

View file

@ -0,0 +1,4 @@
# Fix for League of Legends Updater
location ~ ^.+(releaselisting_.*|.version$) {
proxy_pass http://$host;
}

View file

@ -0,0 +1,6 @@
# Fix for GW2 manifest
location ^~ /latest64 {
proxy_cache_bypass 1;
proxy_no_cache 1;
proxy_pass http://$host$request_uri;
}

View file

@ -0,0 +1,6 @@
# Fix for WSUS authroot cab files
location ~* (authrootstl.cab|pinrulesstl.cab|disallowedcertstl.cab)$ {
proxy_cache_bypass 1;
proxy_no_cache 1;
proxy_pass http://$host$request_uri;
}

View file

@ -0,0 +1,4 @@
location = /server-status {
proxy_no_cache 1;
proxy_cache_bypass 1;
}

View file

@ -0,0 +1,6 @@
location = /lancache-heartbeat {
add_header X-LanCache-Processed-By $hostname;
add_header 'Access-Control-Expose-Headers' '*';
add_header 'Access-Control-Allow-Origin' '*';
return 204;
}

View file

@ -0,0 +1,7 @@
# Abort any circular requests
if ($http_X_LanCache_Processed_By = $hostname) {
return 508;
}
proxy_set_header X-LanCache-Processed-By $hostname;
add_header X-LanCache-Processed-By $hostname,$http_X_LanCache_Processed_By;

View file

@ -0,0 +1,35 @@
# Cache Location
slice 1m;
proxy_cache generic;
proxy_ignore_headers Expires Cache-Control;
proxy_cache_valid 200 206 CACHE_MAX_AGE;
proxy_set_header Range $slice_range;
# Only download one copy at a time and use a large timeout so
# this really happens, otherwise we end up wasting bandwith
# getting the file multiple times.
proxy_cache_lock on;
# If it's taken over a minute to download a 1m file, we are probably stuck!
# Allow the next request to cache
proxy_cache_lock_age 2m;
# If it's totally broken after an hour, stick it in bypass (this shouldn't ever trigger)
proxy_cache_lock_timeout 1h;
# Allow the use of state entries
proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504;
# Allow caching of 200 but not 301 or 302 as our cache key may not include query params
# hence may not be valid for all users
proxy_cache_valid 301 302 0;
# Enable cache revalidation
proxy_cache_revalidate on;
# Don't cache requests marked as nocache=1
proxy_cache_bypass $arg_nocache;
# 40G max file
proxy_max_temp_file_size 40960m;

View file

@ -0,0 +1,3 @@
# Battle.net Fix
proxy_hide_header ETag;

View file

@ -0,0 +1,10 @@
# Upstream Configuration
proxy_next_upstream error timeout http_404;
proxy_pass http://$host$request_uri;
proxy_redirect off;
proxy_ignore_client_abort on;
# Upstream request headers
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

View file

@ -0,0 +1,4 @@
# Debug Headers
add_header X-Upstream-Status $upstream_status;
add_header X-Upstream-Response-Time $upstream_response_time;
add_header X-Upstream-Cache-Status $upstream_cache_status;

View file

@ -0,0 +1,9 @@
server {
listen 443;
resolver UPSTREAM_DNS ipv6=off;
proxy_pass $ssl_preread_server_name:443;
ssl_preread on;
access_log /data/logs/stream-access.log stream_basic;
error_log /data/logs/stream-error.log;
}

View file

@ -0,0 +1 @@
worker_processes 16;

View file

@ -0,0 +1,4 @@
[program:heartbeat]
command=/scripts/heartbeat.sh
stdout_events_enabled=true
stderr_events_enabled=true

View file

@ -7,3 +7,9 @@ if [[ ! -z "${CACHE_DOMAIN_REPO}" ]]; then
fi fi
if [[ ! -z "${CACHE_INDEX_SIZE}" ]]; then
echo " *** CACHE_MEM_SIZE has been deprecated in place of CACHE_INDEX_SIZE"
echo " *** Using CACHE_MEM_SIZE as the value"
echo " *** Please update your configuration at your earliest convenience"
CACHE_INDEX_SIZE=$CACHE_MEM_SIZE
fi

View file

@ -0,0 +1,46 @@
#!/bin/bash
echo "Checking cache configuration"
print_confighash_warning () {
echo ""
echo "ABORTING STARTUP TO AVOID POTENTIALLY INVALIDATING THE CACHE"
echo ""
echo "If you are happy that this cache is valid with the current config changes"
echo "please delete \`/<cache_mount>/CONFIGHASH\`"
echo ""
echo "See: https://lancache.net/docs/advanced/config-hash/ for more details"
}
DETECTED_CACHE_KEY=`grep proxy_cache_key /etc/nginx/sites-available/cache.conf.d/root/30_cache_key.conf | awk '{print $2}'`
NEWHASH="GENERICCACHE_VERSION=${GENERICCACHE_VERSION};CACHE_MODE=${CACHE_MODE};CACHE_SLICE_SIZE=${CACHE_SLICE_SIZE};CACHE_KEY=${DETECTED_CACHE_KEY}"
if [ -d /data/cache/cache ]; then
echo " Detected existing cache data, checking config hash for consistency"
if [ -f /data/cache/CONFIGHASH ]; then
OLDHASH=`cat /data/cache/CONFIGHASH`
if [ ${OLDHASH} != ${NEWHASH} ]; then
echo "ERROR: Detected CONFIGHASH does not match current CONFIGHASH"
echo " Detected: ${OLDHASH}"
echo " Current: ${NEWHASH}"
print_confighash_warning ${NEWHASH}
exit -1;
else
echo " CONFIGHASH matches current configuration"
fi
else
echo " Could not find CONFIGHASH for existing cachedata"
echo " This is either an upgrade from an older instance of Lancache"
echo " or CONFIGHASH has been deleted intentionally"
echo ""
echo " Creating CONFIGHASH from current live configuration"
echo " Current: ${NEWHASH}"
echo ""
echo " See: https://lancache.net/docs/advanced/config-hash/ for more details"
fi
fi
mkdir -p /data/cache/cache
echo ${NEWHASH} > /data/cache/CONFIGHASH

View file

@ -0,0 +1,19 @@
#!/bin/sh
set -e
# Handle CACHE_MEM_SIZE deprecation
if [[ ! -z "${CACHE_MEM_SIZE}" ]]; then
CACHE_INDEX_SIZE=${CACHE_MEM_SIZE}
fi
# Preprocess UPSTREAM_DNS to allow for multiple resolvers using the same syntax as lancache-dns
UPSTREAM_DNS="$(echo -n "${UPSTREAM_DNS}" | sed 's/[;]/ /g')"
echo "worker_processes ${NGINX_WORKER_PROCESSES};" > /etc/nginx/workers.conf
sed -i "s/^user .*/user ${WEBUSER};/" /etc/nginx/nginx.conf
sed -i "s/CACHE_INDEX_SIZE/${CACHE_INDEX_SIZE}/" /etc/nginx/conf.d/20_proxy_cache_path.conf
sed -i "s/CACHE_DISK_SIZE/${CACHE_DISK_SIZE}/" /etc/nginx/conf.d/20_proxy_cache_path.conf
sed -i "s/CACHE_MAX_AGE/${CACHE_MAX_AGE}/" /etc/nginx/sites-available/cache.conf.d/root/20_cache.conf
sed -i "s/slice 1m;/slice ${CACHE_SLICE_SIZE};/" /etc/nginx/sites-available/cache.conf.d/root/20_cache.conf
sed -i "s/UPSTREAM_DNS/${UPSTREAM_DNS}/" /etc/nginx/sites-available/cache.conf.d/10_root.conf
sed -i "s/UPSTREAM_DNS/${UPSTREAM_DNS}/" /etc/nginx/stream-available/10_sni.conf

View file

@ -0,0 +1,14 @@
#!/bin/bash
if [ -d "/data/cache/cache" ]; then
echo "Running fast permissions check"
ls -l /data/cache/cache | tail --lines=+2 | grep -v ${WEBUSER} > /dev/null
if [[ $? -eq 0 || "$FORCE_PERMS_CHECK" == "true" ]]; then
echo "Doing full checking of permissions (This WILL take a long time on large caches)..."
find /data \! -user ${WEBUSER} -exec chown ${WEBUSER}:${WEBUSER} '{}' +
echo "Permissions ok"
else
echo "Fast permissions check successful, if you have any permissions error try running with -e FORCE_PERMS_CHECK = true"
fi
fi

View file

@ -0,0 +1,9 @@
#!/bin/sh
set -e
echo "Currently configured config:"
/scripts/getconfig.sh /etc/nginx/nginx.conf
echo "Checking nginx config"
/usr/sbin/nginx -t
[ $? -ne 0 ] || echo "Config check successful"

29
overlay/scripts/cache_test.sh Executable file
View file

@ -0,0 +1,29 @@
#!/bin/bash
set -e
pageload1=`curl http://www.worldtimeapi.org/api/timezone/ETC/GMT --resolve www.worldtimeapi.org:80:127.0.0.1`
sleep 5
pageload2=`curl http://www.worldtimeapi.org/api/timezone/ETC/GMT --resolve www.worldtimeapi.org:80:127.0.0.1`
sleep 5
pageload3=`curl http://worldtimeapi.org/api/timezone/ETC/GMT --resolve worldtimeapi.org:80:127.0.0.1`
sleep 5
pageload4=`curl http://worldtimeapi.org/api/timezone/ETC/GMT --resolve worldtimeapi.org:80:127.0.0.1`
if [ "$pageload1" == "$pageload2" ]; then
if [ "$pageload3" == "$pageload4" ]; then
if [ "$pageload1" == "$pageload4" ]; then
#In monolithic pages 1+3 should be different as there is no map for this test case
echo "Error caching test page, pages 1+3 are identical"
exit -3
else
echo "Succesfully Cached"
exit 0
fi
else
echo "Error caching test page, pages 3+4 differed"
exit -2
fi
else
echo "Error caching test page, pages 1+2 differed"
exit -1
fi

39
overlay/scripts/getconfig.sh Executable file
View file

@ -0,0 +1,39 @@
#!/bin/bash
get_file_contents() {
FILES=$1
local FILE;
for FILE in $FILES; do
echo "# Including $FILE"
local LINE
while read LINE; do
CLEANLINE=`echo $LINE | sed -e 's/^[[:space:]]*//g' -e 's/[[:space:]]*\$//g'`
if [[ "$CLEANLINE" =~ ^include ]]; then
local CL_LEN
local INCUDE
CL_LEN=${#CLEANLINE}-9;
INCLUDE=${CLEANLINE:8:$CL_LEN}
get_file_contents "$INCLUDE"
else
echo $LINE
fi
done < $FILE
echo "# Finished including $FILE"
done
}
main() {
echo "NGINX CONFIG DUMP FOR $1"
cd `dirname $1`
get_file_contents $1
}
main `readlink -f $1`

16
overlay/scripts/heartbeat.sh Executable file
View file

@ -0,0 +1,16 @@
#!/bin/bash
set -e
if [[ "$1" == "" ]]; then
BEATTIME=${BEAT_TIME}
else
BEATTIME=$1
if [[ "$1" == 0 ]]; then
exit 0;
fi
fi
while [ 1 ]; do
sleep $BEATTIME;
wget http://127.0.0.1/lancache-heartbeat -S -O - > /dev/null 2>&1 /dev/null
done

View file

@ -1,68 +1,9 @@
#!/bin/bash #!/bin/bash
which goss
if [ $? -ne 0 ]; then if [[ "$@" == *" -- "* ]]; then
echo "Please install goss from https://goss.rocks/install" SD_LOGLEVEL="-e SUPERVISORD_LOGLEVEL=INFO"
echo "For a quick auto install run the following" else
echo "curl -fsSL https://goss.rocks/install | sh" SD_LOGLEVEL="-- -e SUPERVISORD_LOGLEVEL=INFO"
exit $?
fi fi
GOSS_WAIT_OPS="-r 60s -s 1s" curl -fsSL https://raw.githubusercontent.com/lancachenet/test-suite/master/dgoss-tests.sh | bash -s -- --imagename="lancachenet/monolithic:goss-test" $@ $SD_LOGLEVEL
docker build --tag lancachenet/monolithic:goss-test .
case $1 in
circleci)
shift;
mkdir -p ./reports/goss
if [[ "$1" == "keepimage" ]]; then
KEEPIMAGE=true
shift
fi
export GOSS_OPTS="$GOSS_OPTS --format junit"
export CONTAINER_LOG_OUTPUT="reports/goss/docker.log"
dgoss run $@ lancachenet/monolithic:goss-test > reports/goss/report.xml
#store result for exit code
RESULT=$?
#Ensure non blank docker.log
echo \
"Container Output:
$(cat reports/goss/docker.log)" \
> reports/goss/docker.log
#delete the junk that goss currently outputs :(
sed -i '0,/^</d' reports/goss/report.xml
#remove invalid system-err outputs from junit output so circleci can read it
sed -i '/<system-err>.*<\/system-err>/d' reports/goss/report.xml
;;
docker)
shift;
if [[ "$1" == "keepimage" ]]; then
KEEPIMAGE=true
shift
fi
docker run --name monolithic-goss-test $@ lancachenet/monolithic:goss-test
docker stop monolithic-goss-test
docker rm monolithic-goss-test
RESULT=$?
;;
edit)
shift;
if [[ "$1" == "keepimage" ]]; then
KEEPIMAGE=true
shift
fi
dgoss edit $@ lancachenet/monolithic:goss-test
RESULT=$?
;;
*)
if [[ "$1" == "keepimage" ]]; then
KEEPIMAGE=true
shift
fi
dgoss run $@ lancachenet/monolithic:goss-test
RESULT=$?
;;
esac
[[ "$KEEPIMAGE" == "true" ]] || docker rmi lancachenet/monolithic:goss-test
exit $RESULT