feat: support WORLD files in compressed tar format (#1297)

.github/workflows/discord.yml

@@ -2,7 +2,7 @@ name: discord
 
 on:
   workflow_run:
-    workflows: ["ContinuousIntegration", "PullRequest", "Build and Publish", "Build and publish multiarch" ]
+    workflows: ["ContinuousIntegration", "Build and Publish", "Build and publish multiarch" ]
     types:
       - completed
 

Dockerfile

@@ -6,6 +6,7 @@ RUN apt-get update \
   && DEBIAN_FRONTEND=noninteractive \
   apt-get install -y \
   imagemagick \
+  file \
   gosu \
   sudo \
   net-tools \
@@ -60,7 +61,7 @@ RUN easy-add --var os=${TARGETOS} --var arch=${TARGETARCH}${TARGETVARIANT} \
   --var version=0.1.1 --var app=maven-metadata-release --file {{.app}} \
   --from https://github.com/itzg/{{.app}}/releases/download/{{.version}}/{{.app}}_{{.version}}_{{.os}}_{{.arch}}.tar.gz
 
-ARG MC_HELPER_VERSION=1.11.0
+ARG MC_HELPER_VERSION=1.16.0
 ARG MC_HELPER_BASE_URL=https://github.com/itzg/mc-image-helper/releases/download/v${MC_HELPER_VERSION}
 RUN curl -fsSL ${MC_HELPER_BASE_URL}/mc-image-helper-${MC_HELPER_VERSION}.tgz \
   | tar -C /usr/share -zxf - \

README.md

@@ -829,23 +829,19 @@ before unpacking new content from the MODPACK or MODS.
 
 ### Downloadable world
 
-Instead of mounting the `/data` volume, you can instead specify the URL of a ZIP file containing an archived world. It will be searched for a file `level.dat` and the containing subdirectory moved to the directory named by `$LEVEL`. This means that most of the archived Minecraft worlds downloadable from the Internet will already be in the correct format.
+Instead of mounting the `/data` volume, you can instead specify the URL of a ZIP or compressed TAR file containing an archived world. It will be searched for a file `level.dat` and the containing subdirectory moved to the directory named by `$LEVEL`. This means that most of the archived Minecraft worlds downloadable from the Internet will already be in the correct format.
 
     docker run -d -e WORLD=http://www.example.com/worlds/MySave.zip ...
 
-**NOTE:** This URL must be accessible from inside the container. Therefore,
-you should use an IP address or a globally resolvable FQDN, or else the
-name of a linked container.
+**NOTE:** This URL must be accessible from inside the container. Therefore, you should use an IP address or a globally resolvable FQDN, or else the name of a linked container.
 
 **NOTE:** If the archive contains more than one `level.dat`, then the one to select can be picked with `WORLD_INDEX`, which defaults to 1.
 
 ### Cloning world from a container path
 
-The `WORLD` option can also be used to reference a directory or zip file that will be used as a source to clone or unzip the world directory.
+The `WORLD` option can also be used to reference a directory, zip file, or compressed tar file that will be used as a source to clone or extract the world directory.
 
-For example, the following would initially clone the world's content
-from `/worlds/basic`. Also notice in the example that you can use a
-read-only volume attachment to ensure the clone source remains pristine.
+For example, the following would initially clone the world's content from `/worlds/basic`. Also notice in the example that you should use a read-only volume attachment to ensure the clone source remains pristine.
 
 ```
 docker run ... -v $HOME/worlds:/worlds:ro -e WORLD=/worlds/basic

scripts/start-setupWorld

@@ -5,7 +5,7 @@
 set -e
 isDebugging && set -x
 
-if [ $TYPE = "CURSEFORGE" ]; then
+if [ "$TYPE" = "CURSEFORGE" ]; then
   worldDest=$FTB_DIR/${LEVEL:-world}
 else
   worldDest=/data/${LEVEL:-world}
@@ -19,19 +19,23 @@ if [[ "$WORLD" ]] && ( isTrue "${FORCE_WORLD_COPY}" || [ ! -d "$worldDest" ] );
       "${worldDest}_the_end"
   fi
 
-  if isURL $WORLD; then
-    curl -fsSL "$WORLD" -o /tmp/world.zip
-    zipSrc=/tmp/world.zip
-  elif [[ "$WORLD" =~ .*\.zip ]]; then
-    zipSrc="$WORLD"
+  if isURL "$WORLD"; then
+    log "Downloading world from $WORLD"
+    if ! get -o /tmp/world.bin "$WORLD"; then
+      log "ERROR: failed to download world from $WORLD"
+      exit 1
+    fi
+    WORLD=/tmp/world.bin
   fi
 
-  if [[ "$zipSrc" ]]; then
-    log "Unzipping world"
+  if [ -f "$WORLD" ]; then
+    log "Extracting world"
 
     # Stage contents so that the correct subdirectory can be picked off
     mkdir -p /tmp/world-data
-    (cd /tmp/world-data && unzip -o -q "$zipSrc")
+    if ! extract "$WORLD" /tmp/world-data; then
+      exit 1
+    fi
 
     if [ "$FAMILY" = "SPIGOT" ]; then
       baseDirs=$(find /tmp/world-data -name "level.dat" -not -path "*_nether*" -not -path "*_the_end*" -exec dirname "{}" \;)
@@ -39,6 +43,11 @@ if [[ "$WORLD" ]] && ( isTrue "${FORCE_WORLD_COPY}" || [ ! -d "$worldDest" ] );
       baseDirs=$(find /tmp/world-data -name "level.dat" -exec dirname "{}" \;)
     fi
 
+    if ! [[ $baseDirs ]]; then
+      log "ERROR world content is not valid since level.dat could not be found"
+      exit 2
+    fi
+
     count=$(echo "$baseDirs" | wc -l)
     if [[ $count -gt 1 ]]; then
       baseDir="$(echo "$baseDirs" | sed -n ${WORLD_INDEX:-1}p)"
@@ -56,9 +65,12 @@ if [[ "$WORLD" ]] && ( isTrue "${FORCE_WORLD_COPY}" || [ ! -d "$worldDest" ] );
       [ -d "${baseDir}_nether" ] && rsync --remove-source-files --recursive --delete "${baseDir}_nether/" "${worldDest}_nether"
       [ -d "${baseDir}_the_end" ] && rsync --remove-source-files --recursive --delete "${baseDir}_the_end/" "${worldDest}_the_end"
     fi
-  else
+  elif [ -d "$WORLD" ]; then
     log "Cloning world directory from $WORLD ..."
     rsync --recursive --delete "${WORLD%/}"/ "$worldDest"
+  else
+    log "ERROR: world file/directory $WORLD is missing"
+    exit 1
   fi
 
   if [ "$FAMILY" = "SPIGOT" ]; then

scripts/start-utils

@@ -188,4 +188,19 @@ function isType() {
     fi
   done
   return 1
+}
+
+function extract() {
+  src=${1?}
+  destDir=${2?}
+
+  type=$(file -b --mime-type "${src}")
+  if [[ $type == application/zip ]]; then
+    unzip -q -d "${destDir}" "${src}"
+  elif [[ $type == application/x-tar ]]; then
+    tar -C "${destDir}" -xf "${src}"
+  else
+    log "ERROR: unsupported archive type: $type"
+    return 1
+  fi
 }

tests/.gitignore

@@ -0,0 +1 @@
+data/

tests/setuponlytests/generic-packs/docker-compose.yml

@@ -9,6 +9,4 @@ services:
       GENERIC_PACKS: https://github.com/itzg/mc-image-helper/releases/download/v1.9.5/mc-image-helper-1.9.5.zip,/packs/testing.zip
     volumes:
       - ./packs:/packs
-      - data:/data
-volumes:
-  data: {}
+      - ./data:/data

tests/setuponlytests/generic-packs/verify.sh

@@ -0,0 +1 @@
+mc-image-helper assert fileExists one.txt mods/two.txt

tests/setuponlytests/test.sh

@@ -1,43 +1,50 @@
 #!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
 
 # go to script root directory
 cd "$(dirname "$0")" || exit 1
 
-# compose down function for reuse
-down() {
-  docker-compose down -v --remove-orphans
-}
-
-checkandExitOnFailure(){
-  failed=$1
-  # docker-compose logs outputs messages from the specified container
-  if $failed; then
-    docker-compose logs mc
-    down
-    cd ..
-    exit 2
-  fi
-}
-
-# tests that only run the setup files for things like downloads and configuration. 
+# tests that only run the setup files for things like downloads and configuration.
 setupOnlyMinecraftTest(){
   folder=$1
   cd "$folder"
-  failed=false
-  # run the monitor to validate the Minecraft image is healthy
-  docker-compose --log-level ERROR up --quiet-pull --exit-code-from mc 2>/dev/null || failed=true
-  echo "${folder} Result: failed=$failed"
-  checkandExitOnFailure $failed
-  down
+  result=0
+
+  if ! logs=$(docker compose run --quiet-pull mc 2>&1); then
+    echo "${folder} setup FAILED"
+    echo ":::::::::::: LOGS ::::::::::::::::
+$logs
+::::::::::::::::::::::::::::::::::
+"
+    result=1
+  elif [ -f verify.sh ]; then
+    if ! docker run --rm --entrypoint bash -v "${PWD}/data":/data -v "${PWD}/verify.sh":/verify "${IMAGE_TO_TEST:-itzg/minecraft-server}" /verify; then
+      echo "${folder} verify FAILED"
+      result=1
+    else
+      echo "${folder} verify PASS"
+    fi
+  else
+    echo "${folder} PASS"
+  fi
+
+  docker compose down -v --remove-orphans
   cd ..
+
+  return $result
 }
 
 # go through each folder in setuponly and test setups
-FOLDERS=$(ls)
-for folder in $FOLDERS; do
-    # If folder is a directory
-    if [ -d "$folder" ]; then
-      echo "Starting Tests on ${folder}"
-      setupOnlyMinecraftTest $folder
-    fi
-done
+if (( $# > 0 )); then
+  for folder in "$@"; do
+    echo "Starting Tests in ${folder}"
+    setupOnlyMinecraftTest "$folder"
+  done
+else
+  readarray -t folders < <(find . -maxdepth 2 -mindepth 2 -name docker-compose.yml -printf '%h\n')
+  for folder in "${folders[@]}"; do
+    echo "Starting Tests in ${folder}"
+    setupOnlyMinecraftTest "$folder"
+  done
+fi

tests/setuponlytests/world_from_tgz/docker-compose.yml

@@ -0,0 +1,14 @@
+version: "3"
+
+services:
+  mc:
+    restart: "no"
+    image: ${IMAGE_TO_TEST:-itzg/minecraft-server}
+    environment:
+      EULA: "TRUE"
+      SETUP_ONLY: "TRUE"
+      VERSION: ${MINECRAFT_VERSION:-LATEST}
+      WORLD: /worlds/world-for-testing.tgz
+    volumes:
+      - ./worlds:/worlds:ro
+      - ./data:/data

tests/setuponlytests/world_from_tgz/verify.sh

@@ -0,0 +1 @@
+mc-image-helper assert fileExists world/level.dat

tests/setuponlytests/world_from_zip/docker-compose.yml

@@ -0,0 +1,14 @@
+version: "3"
+
+services:
+  mc:
+    restart: "no"
+    image: ${IMAGE_TO_TEST:-itzg/minecraft-server}
+    environment:
+      EULA: "TRUE"
+      SETUP_ONLY: "TRUE"
+      VERSION: ${MINECRAFT_VERSION:-LATEST}
+      WORLD: /worlds/world-for-testing.zip
+    volumes:
+      - ./worlds:/worlds:ro
+      - ./data:/data

tests/setuponlytests/world_from_zip/verify.sh

@@ -0,0 +1 @@
+mc-image-helper assert fileExists world/level.dat

tests/test.sh

@@ -1,18 +1,15 @@
 #!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
 
 # go to script root directory
 cd "$(dirname "$0")" || exit 1
 
 # go through top level folders and trigger the tests in the subfolders
-FOLDERS=$(ls)
-for folder in $FOLDERS; do
-    # If folder is a directory
-    if [ -d "$folder" ]; then
-      cd "$folder"
-      if [ -f "./test.sh" ]; then
-        echo "Starting ${folder} Tests"
-        sh ./test.sh
-      fi
-      cd ..
-    fi
+readarray -t folders < <(find . -maxdepth 2 -mindepth 2 -name test.sh -printf '%h\n')
+for folder in "${folders[@]}"; do
+  cd "$folder"
+  echo "Starting ${folder} Tests"
+  bash ./test.sh
+  cd ..
 done