Rebase generic off monolithic

This commit rebases the generic code into the monolithic image
This allows the base image to be the more regularly accessed / edited image

Commits to lancachenet/generic:switcheroo will mimic this commit

Also fixes lancachenet/generic#108

.circleci/config.yml

@@ -1,5 +1,5 @@
 workflows:
-  version: 2
+  version: 2.1
   build_test_deploy:
     jobs:
       - test
@@ -11,22 +11,26 @@ workflows:
              branches:
                only: 
                  - master
+      - build_children:
+          context: circle-api
+          requires:
+            - publish_latest
 
-version: 2
+version: 2.1
+executors:
+  testbuild-executor:
+    machine:
+      image: ubuntu-1604:201903-01
 jobs:
   test:
-    docker:
+    executor: testbuild-executor
-      - image: circleci/python:2-jessie
     steps:
       - checkout
-
-      - setup_remote_docker:   # (2)
-          docker_layer_caching: false # (3)
       - run:
           name: Install goss
           command: |
             # rather than give internet scripts SU rights, we install to local user bin and add to path
-            mkdir ~/bin
+            [ -d ~/bin ] || mkdir ~/bin
             export GOSS_DST=~/bin
             export PATH=$PATH:~/bin
             curl -fsSL https://goss.rocks/install | sh
@@ -36,39 +40,45 @@ jobs:
           command: |
             # Don't forget path!
             export PATH=$PATH:~/bin
-            # Important, change from mount to work on remote docker, see https://github.com/aelsabbahy/goss/pull/271
+            ./run-tests.sh --circleci --keepimage
-            # If using machine image you do not need this.
-            export GOSS_FILES_STRATEGY=cp
-            ./run-tests.sh circleci keepimage
       - run:
           name: Save docker image
           command: |
-            mkdir -p workspace
+            [ -d workspace ] || mkdir workspace
             docker save -o workspace/lancachenet-monolithic.tar lancachenet/monolithic:goss-test
-      - persist_to_workspace:
+            #Download from Artifacts and Load this into your own docker using the following command
-          root: workspace
+            #docker load -i /tmp/workspace/lancachenet-monolithic.tar
-          paths:
-            lancachenet-monolithic.tar
       - store_test_results:
-          path: reports
+          path: reports/goss/report.xml
       - store_artifacts:
           path: reports
           destination: reports
       - store_artifacts:
           path: workspace/lancachenet-monolithic.tar
           destination: docker-lancachenet-monolithic.tar
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            lancachenet-monolithic.tar
   publish_latest:
-    docker:
+    executor: testbuild-executor
-      - image: circleci/python:2-jessie
     steps:
-      - setup_remote_docker:   # (2)
-          docker_layer_caching: false # (3)
       - attach_workspace:
           at: /tmp/workspace
       - run:
           name: "Deploy latest to docker hub"
           command: |
-            docker login -u $DOCKER_USER -p $DOCKER_PASS
             docker load -i /tmp/workspace/lancachenet-monolithic.tar
+            docker login -u $DOCKER_USER -p $DOCKER_PASS
             docker tag lancachenet/monolithic:goss-test lancachenet/monolithic:latest
             docker push lancachenet/monolithic:latest
+  build_children:
+    executor: testbuild-executor
+    steps:
+      - run:
+          name: "Request API to build children"
+          command: |
+              for child in {"monolithic"}; do
+                 echo "Asking API to trigger build for $child"
+                 curl -X POST --header "Content-Type: application/json" -d '{"branch":"master"}' https://circleci.com/api/v1.1/project/github/lancachenet/$child/build?circle-token=${CIRCLE_API_USER_TOKEN}
+              done

.gitignore

@@ -0,0 +1 @@
+reports

Dockerfile

@@ -1,5 +1,7 @@
-FROM lancachenet/generic:latest
+FROM lancachenet/ubuntu-nginx:latest
-MAINTAINER LanCache.Net Team <team@lancache.net>
+LABEL version=3
+LABEL description="Single caching container for caching game content at lan parties."
+LABEL maintainer=" LanCache.Net Team <team@lancache.net>"
 
 ENV GENERICCACHE_VERSION=2 \
     CACHE_MODE=monolithic \
@@ -17,6 +19,20 @@ ENV GENERICCACHE_VERSION=2 \
 
 COPY overlay/ /
 
+RUN rm /etc/nginx/sites-enabled/* /etc/nginx/stream-enabled/* ;\
+    rm /etc/nginx/conf.d/gzip.conf ;\
+    chmod 754  /var/log/tallylog ; \
+    id -u ${WEBUSER} &> /dev/null || adduser --system --home /var/www/ --no-create-home --shell /bin/false --group --disabled-login ${WEBUSER} ;\
+    chmod 755 /scripts/*			;\
+	mkdir -m 755 -p /data/cache		;\
+	mkdir -m 755 -p /data/info		;\
+	mkdir -m 755 -p /data/logs		;\
+	mkdir -m 755 -p /tmp/nginx/		;\
+	chown -R ${WEBUSER}:${WEBUSER} /data/	;\
+	mkdir -p /etc/nginx/sites-enabled	;\
+	ln -s /etc/nginx/sites-available/10_cache.conf /etc/nginx/sites-enabled/10_generic.conf; \
+	ln -s /etc/nginx/stream-available/10_sni.conf /etc/nginx/stream-enabled/10_sni.conf
+
 RUN mkdir -m 755 -p /data/cachedomains		;\
 	mkdir -m 755 -p /tmp/nginx				;\
 	apt-get update							;\
@@ -26,5 +42,5 @@ RUN git clone --depth=1 --no-single-branch https://github.com/uklans/cache-domai
 
 VOLUME ["/data/logs", "/data/cache", "/data/cachedomains", "/var/www"]
 
-EXPOSE 80
+EXPOSE 80 443
 WORKDIR /scripts

goss.yaml

@@ -13,7 +13,7 @@ command:
     exit-status: 0
     stdout:
     - Succesfully Cached
-    timeout: 10000
+    timeout: 20000
 process:
   nginx:
     running: true

overlay/etc/nginx/conf.d/20_proxy_cache_path.conf

@@ -0,0 +1 @@
+proxy_cache_path /data/cache/cache levels=2:2 keys_zone=generic:CACHE_MEM_SIZE inactive=200d max_size=CACHE_DISK_SIZE loader_files=1000 loader_sleep=50ms loader_threshold=300ms use_temp_path=off;

overlay/etc/nginx/nginx.conf

@@ -0,0 +1,39 @@
+user www-data;
+include /etc/nginx/workers.conf;
+pid /run/nginx.pid;
+
+include /etc/nginx/modules-enabled/*.conf;
+
+events {
+	worker_connections 4096;
+	multi_accept on;
+	use epoll;
+}
+
+http {
+	aio threads;
+	sendfile on;
+	tcp_nopush on;
+	tcp_nodelay on;
+	keepalive_timeout 65;
+	types_hash_max_size 2048;
+
+	include /etc/nginx/mime.types;
+	default_type application/octet-stream;
+
+	access_log /var/log/nginx/access.log;
+	error_log /var/log/nginx/error.log;
+
+
+	gzip on;
+
+	include /etc/nginx/conf.d/*.conf;
+
+	include /etc/nginx/sites-enabled/*.conf;
+}
+
+
+stream {                                
+	include /etc/nginx/stream.d/*.conf; 
+	include /etc/nginx/stream-enabled/*;
+}                                       

overlay/etc/nginx/sites-available/10_cache.conf

@@ -0,0 +1,10 @@
+
+server {
+  listen 80 reuseport;
+
+  access_log /data/logs/access.log cachelog;
+  error_log /data/logs/error.log;
+
+
+  include /etc/nginx/sites-available/cache.conf.d/*.conf;
+}

overlay/etc/nginx/sites-available/cache.conf.d/10_root.conf

@@ -0,0 +1,8 @@
+  resolver UPSTREAM_DNS ipv6=off;
+
+  location / {
+
+    include /etc/nginx/sites-available/cache.conf.d/root/*.conf;
+
+  }
+

overlay/etc/nginx/sites-available/cache.conf.d/20_lol.conf

@@ -0,0 +1,5 @@
+
+  # Fix for League of Legends Updater
+  location ~ ^.+(releaselisting_.*|.version$) {
+    proxy_pass http://$host;
+  }

overlay/etc/nginx/sites-available/cache.conf.d/21_arenanet_manifest.conf

@@ -0,0 +1,6 @@
+  # Fix for GW2 manifest
+  location ^~ /latest64 {
+	proxy_cache_bypass 1;
+	proxy_no_cache 1;
+	proxy_pass http://$host$request_uri;
+  }

overlay/etc/nginx/sites-available/cache.conf.d/22_wsus_cabs.conf

@@ -0,0 +1,6 @@
+ # Fix for WSUS authroot cab files
+  location ~* (authrootstl.cab|pinrulesstl.cab|disallowedcertstl.cab)$ {
+	proxy_cache_bypass 1;
+	proxy_no_cache 1;
+	proxy_pass http://$host$request_uri;
+  }

overlay/etc/nginx/sites-available/cache.conf.d/90_lancache_heartbeat.conf

@@ -0,0 +1,6 @@
+  location = /lancache-heartbeat {
+    add_header X-LanCache-Processed-By $hostname;
+    add_header 'Access-Control-Expose-Headers' '*';
+    add_header 'Access-Control-Allow-Origin' '*';
+    return 204;
+  }

overlay/etc/nginx/sites-available/cache.conf.d/root/10_loop_detection.conf

@@ -0,0 +1,7 @@
+    # Abort any circular requests
+	if ($http_X_LanCache_Processed_By = $hostname) {
+		return 508;
+	}
+
+	proxy_set_header X-LanCache-Processed-By $hostname;
+	add_header X-LanCache-Processed-By $hostname,$http_X_LanCache_Processed_By;

overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf

@@ -0,0 +1,30 @@
+    # Cache Location
+    slice 1m;
+    proxy_cache generic;
+
+    proxy_ignore_headers Expires Cache-Control;
+    proxy_cache_valid 200 206 CACHE_MAX_AGE;
+    proxy_set_header  Range $slice_range;
+
+    # Only download one copy at a time and use a large timeout so
+    # this really happens, otherwise we end up wasting bandwith
+    # getting the file multiple times.
+    proxy_cache_lock on;
+    proxy_cache_lock_timeout 1h;
+
+    # Allow the use of state entries
+    proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504;
+
+    # Allow caching of 200 but not 301 or 302 as our cache key may not include query params
+    # hence may not be valid for all users
+    proxy_cache_valid 301 302 0;
+
+    # Enable cache revalidation
+    proxy_cache_revalidate on;
+    
+    # Don't cache requests marked as nocache=1
+    proxy_cache_bypass $arg_nocache;
+
+    # 40G max file
+    proxy_max_temp_file_size 40960m;
+

overlay/etc/nginx/sites-available/cache.conf.d/root/40_etags.conf

@@ -0,0 +1,3 @@
+    # Battle.net Fix
+    proxy_hide_header ETag;
+

overlay/etc/nginx/sites-available/cache.conf.d/root/90_upstream.conf

@@ -0,0 +1,10 @@
+    # Upstream Configuration
+    proxy_next_upstream error timeout http_404;
+    proxy_pass http://$host$request_uri;
+    proxy_redirect off;
+    proxy_ignore_client_abort on;
+
+    # Upstream request headers
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

overlay/etc/nginx/sites-available/cache.conf.d/root/99_debug_header.conf

@@ -0,0 +1,4 @@
+    # Debug Headers
+    add_header X-Upstream-Status $upstream_status;
+    add_header X-Upstream-Response-Time $upstream_response_time;
+    add_header X-Upstream-Cache-Status $upstream_cache_status;

overlay/etc/nginx/stream-available/10_sni.conf

@@ -0,0 +1,9 @@
+server {
+	listen 443;
+    resolver UPSTREAM_DNS ipv6=off;
+	proxy_pass  $ssl_preread_server_name:443;
+	ssl_preread on;
+
+	access_log /data/logs/stream-access.log stream_basic;
+	error_log /data/logs/stream-error.log;
+}

overlay/etc/nginx/workers.conf

@@ -0,0 +1 @@
+worker_processes 16;

overlay/etc/supervisor/conf.d/heartbeat.conf

@@ -0,0 +1,4 @@
+[program:heartbeat]
+command=/scripts/heartbeat.sh
+stdout_events_enabled=true
+stderr_events_enabled=true

overlay/hooks/entrypoint-pre.d/05_config_check.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+echo "Checking cache configuration"
+
+
+print_confighash_warning () {
+	echo ""
+	echo "ABORTING STARTUP TO AVOID POTENTIALLY INVALIDATING THE CACHE"
+	echo ""
+	echo "If you are happy that this cache is valid with the current config changes"
+	echo "please delete \`/<cache_mount>/CONFIGHASH\`"
+	echo ""
+	echo "See: https://lancache.net/docs/advanced/config-hash/ for more details"
+
+}
+
+DETECTED_CACHE_KEY=`grep proxy_cache_key /etc/nginx/sites-available/cache.conf.d/root/30_cache_key.conf | awk '{print $2}'`
+NEWHASH="GENERICCACHE_VERSION=${GENERICCACHE_VERSION};CACHE_MODE=${CACHE_MODE};CACHE_SLICE_SIZE=${CACHE_SLICE_SIZE};CACHE_KEY=${DETECTED_CACHE_KEY}"
+
+if [ -d /data/cache/cache ]; then
+	echo " Detected existing cache data, checking config hash for consistency"
+	if [ -f /data/cache/CONFIGHASH ]; then
+		OLDHASH=`cat /data/cache/CONFIGHASH`
+		if [ ${OLDHASH} != ${NEWHASH} ]; then
+			echo "ERROR: Detected CONFIGHASH does not match current CONFIGHASH"
+			echo " Detected: ${OLDHASH}"
+			echo " Current:  ${NEWHASH}"
+			print_confighash_warning ${NEWHASH}
+			exit -1;
+		else
+			echo " CONFIGHASH matches current configuration"
+		fi
+	else
+		echo " Could not find CONFIGHASH for existing cachedata"
+		echo "  This is either an upgrade from an older instance of Lancache"
+		echo "  or CONFIGHASH has been deleted intentionally"
+		echo ""
+		echo " Creating CONFIGHASH from current live configuration"
+		echo "   Current:  ${NEWHASH}"
+		echo ""
+		echo "  See: https://lancache.net/docs/advanced/config-hash/ for more details"
+	fi
+fi
+
+mkdir -p /data/cache/cache
+echo ${NEWHASH} > /data/cache/CONFIGHASH

overlay/hooks/entrypoint-pre.d/10_setup.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+set -e
+
+# Preprocess UPSTREAM_DNS to allow for multiple resolvers using the same syntax as lancache-dns
+UPSTREAM_DNS="$(echo -n "${UPSTREAM_DNS}" | sed 's/[;]/ /g')"
+
+echo "worker_processes ${NGINX_WORKER_PROCESSES};" > /etc/nginx/workers.conf
+sed -i "s/^user .*/user ${WEBUSER};/" /etc/nginx/nginx.conf
+sed -i "s/CACHE_MEM_SIZE/${CACHE_MEM_SIZE}/"  /etc/nginx/conf.d/20_proxy_cache_path.conf
+sed -i "s/CACHE_DISK_SIZE/${CACHE_DISK_SIZE}/" /etc/nginx/conf.d/20_proxy_cache_path.conf
+sed -i "s/CACHE_MAX_AGE/${CACHE_MAX_AGE}/"    /etc/nginx/sites-available/cache.conf.d/root/20_cache.conf
+sed -i "s/slice 1m;/slice ${CACHE_SLICE_SIZE};/" /etc/nginx/sites-available/cache.conf.d/root/20_cache.conf
+sed -i "s/UPSTREAM_DNS/${UPSTREAM_DNS}/"    /etc/nginx/sites-available/cache.conf.d/10_root.conf
+sed -i "s/UPSTREAM_DNS/${UPSTREAM_DNS}/"    /etc/nginx/stream-available/10_sni.conf

overlay/hooks/entrypoint-pre.d/20_perms_check.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+if [ -d "/data/cache/cache" ]; then
+	echo "Running fast permissions check"
+	ls -l /data/cache/cache | tail --lines=+2 | grep -v ${WEBUSER} > /dev/null
+
+	if [[ $? -eq 0 || "$FORCE_PERMS_CHECK" == "true" ]]; then
+		echo "Doing full checking of permissions (This WILL take a long time on large caches)..."
+		find /data \! -user ${WEBUSER} -exec chown ${WEBUSER}:${WEBUSER} '{}' +
+		echo "Permissions ok"
+	else
+		echo "Fast permissions check successful, if you have any permissions error try running with -e FORCE_PERMS_CHECK = true"
+	fi
+
+fi

overlay/hooks/supervisord-pre.d/99_config_check.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -e
+echo "Currently configured config:"
+/scripts/getconfig.sh /etc/nginx/nginx.conf
+
+echo "Checking nginx config"
+/usr/sbin/nginx -t
+
+ [ $? -ne 0 ] || echo "Config check successful"

overlay/scripts/cache_test.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+pageload1=`curl http://www.worldtimeapi.org/api/timezone/ETC/GMT --resolve www.worldtimeapi.org:80:127.0.0.1`
+sleep 5
+pageload2=`curl http://www.worldtimeapi.org/api/timezone/ETC/GMT --resolve www.worldtimeapi.org:80:127.0.0.1`
+sleep 5
+pageload3=`curl http://worldtimeapi.org/api/timezone/ETC/GMT --resolve worldtimeapi.org:80:127.0.0.1`
+sleep 5
+pageload4=`curl http://worldtimeapi.org/api/timezone/ETC/GMT --resolve worldtimeapi.org:80:127.0.0.1`
+if [ "$pageload1" == "$pageload2" ]; then
+	if [ "$pageload3" == "$pageload4" ]; then
+		if [ "$pageload1" == "$pageload4" ]; then
+			#In monolithic pages 1+3 should be different as there is no map for this test case
+			echo "Error caching test page, pages 1+3 are identical"
+			exit -3
+		else
+			echo "Succesfully Cached"
+			exit 0
+		fi
+
+	else
+		echo "Error caching test page, pages 3+4 differed"
+		exit -2
+	fi
+
+else
+	echo "Error caching test page, pages 1+2 differed"
+	exit -1
+fi

overlay/scripts/getconfig.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+get_file_contents() {
+	FILES=$1
+	local FILE;
+	for FILE in $FILES; do
+		echo "# Including $FILE"
+		local LINE
+		while read LINE; do
+			CLEANLINE=`echo $LINE | sed -e 's/^[[:space:]]*//g' -e 's/[[:space:]]*\$//g'`
+			if [[ "$CLEANLINE" =~ ^include ]]; then
+				local CL_LEN
+				local INCUDE
+				CL_LEN=${#CLEANLINE}-9;
+				INCLUDE=${CLEANLINE:8:$CL_LEN}
+				get_file_contents "$INCLUDE"
+			else
+				echo $LINE
+			fi
+		done < $FILE
+		echo "# Finished including $FILE"
+
+	done
+}
+
+
+main() {
+
+	echo "NGINX CONFIG DUMP FOR $1"
+
+	cd `dirname $1`
+
+	get_file_contents $1
+
+}
+
+
+
+main `readlink -f $1`

overlay/scripts/heartbeat.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+if [[ "$1" == "" ]]; then
+	BEATTIME=${BEAT_TIME}
+else
+	BEATTIME=$1
+	if [[ "$1" == 0 ]]; then 
+		exit 0;
+	fi
+fi
+
+
+while [ 1 ]; do
+    sleep $BEATTIME;
+	wget http://127.0.0.1/lancache-heartbeat -S -O - > /dev/null 2>&1 /dev/null
+done

run-tests.sh

@@ -1,68 +1,9 @@
 #!/bin/bash
-which goss
 
-if [ $? -ne 0 ]; then
+if [[ "$@" == *" -- "* ]]; then
-	echo "Please install goss from https://goss.rocks/install"
+	SD_LOGLEVEL="-e SUPERVISORD_LOGLEVEL=INFO"
-	echo "For a quick auto install run the following"
+else
-	echo "curl -fsSL https://goss.rocks/install | sh"
+	SD_LOGLEVEL="-- -e SUPERVISORD_LOGLEVEL=INFO"
-	exit $?
 fi
 
-GOSS_WAIT_OPS="-r 60s -s 1s"
+curl -fsSL https://raw.githubusercontent.com/lancachenet/test-suite/master/dgoss-tests.sh | bash -s -- --imagename="lancachenet/monolithic:goss-test" $@ $SD_LOGLEVEL
-
-docker build --tag lancachenet/monolithic:goss-test .
-case $1 in
-  circleci)
-	shift;
-	mkdir -p ./reports/goss
-	if [[ "$1" == "keepimage" ]]; then
-		KEEPIMAGE=true
-		shift
-	fi
-	export GOSS_OPTS="$GOSS_OPTS --format junit"
-	export CONTAINER_LOG_OUTPUT="reports/goss/docker.log"
-	dgoss run $@ lancachenet/monolithic:goss-test > reports/goss/report.xml
-	#store result for exit code
-	RESULT=$?
-	#Ensure non blank docker.log
-	echo \
-"Container Output:
-$(cat reports/goss/docker.log)" \
-	> reports/goss/docker.log
-	#delete the junk that goss currently outputs :(
-	sed -i '0,/^</d' reports/goss/report.xml
-	#remove invalid system-err outputs from junit output so circleci can read it
-	sed -i '/<system-err>.*<\/system-err>/d' reports/goss/report.xml
-    ;;
-  docker)
-	shift;
-	if [[ "$1" == "keepimage" ]]; then
-		KEEPIMAGE=true
-		shift
-	fi
-	docker run --name monolithic-goss-test $@ lancachenet/monolithic:goss-test
-	docker stop monolithic-goss-test
-	docker rm monolithic-goss-test
-	RESULT=$?
-	;;
-  edit)
-	shift;
-	if [[ "$1" == "keepimage" ]]; then
-		KEEPIMAGE=true
-		shift
-	fi
-	dgoss edit $@ lancachenet/monolithic:goss-test
-	RESULT=$?
-    ;;
-  *)
-	if [[ "$1" == "keepimage" ]]; then
-		KEEPIMAGE=true
-		shift
-	fi
-	dgoss run $@ lancachenet/monolithic:goss-test
-	RESULT=$?
-    ;;
-esac
-[[ "$KEEPIMAGE" == "true" ]] || docker rmi lancachenet/monolithic:goss-test
-
-exit $RESULT