ForgeAPI: Random failure fix (#1384)

* Fix for Random forge api failure, removing dependency duplication messages

* pushing array test for review

* Fixing while loops that were causing downloaded modids to reset

Co-authored-by: christopher blodgett <christopher.blodgett@gmail.com>

.github/workflows/build-multiarch.yml

@@ -56,7 +56,7 @@ jobs:
             baseImage: ibm-semeru-runtimes:open-11-jre
             platforms: linux/amd64,linux/arm64
             mcVersion: 1.16.5
-        # JAVA 8: NOTE: Unable to go past 8u312 because of Forge https://github.com/MultiMC/Launcher/issues/447
+        # JAVA 8: NOTE: Unable to go past 8u312 because of Forge dependencies
           - variant: java8
             baseImage: openjdk:8-jre-alpine3.9
             platforms: linux/amd64

.gitignore

@@ -3,3 +3,4 @@
 /.idea/
 *.iml
 /gh-md-toc
+personal-build-and-develop.*

scripts/start-setupForgeApiMods

@@ -17,6 +17,7 @@ FORGEAPI_BASE_URL=${FORGEAPI_BASE_URL:-https://api.curseforge.com/v1}
 RELEASE_NUMBER_FILTER=1
 MINECRAFT_GAME_ID=432
 FILTER_BY_FAMILY=false
+DOWNLOADED_MODIDS=()
 out_dir=/data/mods
 
 # shellcheck source=start-utils
@@ -24,8 +25,8 @@ out_dir=/data/mods
 isDebugging && set -x
 
 # Remove old mods/plugins
-if isTrue "${REMOVE_OLD_FORGEAPI_MODS}" && [ -z "${MODS_FORGEAPI_KEY}" ]; then
-  removeOldMods /data/mods
+if isTrue "${REMOVE_OLD_FORGEAPI_MODS}"; then
+  removeOldMods "/data/mods"
 fi
 
 # Family filter is on by default for Forge, Fabric, and Bukkit
@@ -104,8 +105,8 @@ modFileByProjectID(){
     total_count=$(jq -n "$project_files" | jq -c '.pagination.totalCount' )
 
     # Checking for a individual release type input, if not use global
-    if [ $project_id_release_type ]; then
-      updateReleaseNumber $project_id_release_type
+    if [ "$project_id_release_type" ]; then
+      updateReleaseNumber "$project_id_release_type"
       unset project_id_release_type 
     else
       updateReleaseNumber $MODS_FORGEAPI_RELEASES
@@ -113,7 +114,7 @@ modFileByProjectID(){
 
     # grabs the highest ID of the releaseTypes selected.
     # Default is 1 for Release, Beta is 2, and Alpha is 3. Using less than we can validate highest release.
-    if [ $project_id_file_name ]; then
+    if [ "$project_id_file_name" ]; then
       # Looks for file by name
       current_project_file=$(jq -n "$project_files" | jq --arg FILE_NAME "$project_id_file_name" -jc '
         .data | map(select(.fileName<=($FILE_NAME))) | .[0] // empty')
@@ -131,20 +132,20 @@ modFileByProjectID(){
     if [ ! "$PROJECT_FILE" ]; then
       PROJECT_FILE=$current_project_file
     elif [ "$current_project_file" ]; then
-      current_project_file_id=$(jq -n "$current_project_file" | jq -jc '.id' )
-      PROJECT_FILE_ID=$(jq -n "$PROJECT_FILE" | jq -jc '.id' )
+      current_project_file_id=$(jq -n "$current_project_file" | jq -jc '.id // empty' )
+      PROJECT_FILE_ID=$(jq -n "$PROJECT_FILE" | jq -jc '.id // empty' )
       if (( current_project_file_id > PROJECT_FILE_ID )); then
         PROJECT_FILE=$current_project_file
       fi
     fi
     
     # check to see if we have gone to far or lost our index and exit with an error
-    if [ -z "$index" ] || [ -z "$total_count" ] || [ $index -ge $total_count ]; then
+    if [ -z "$index" ] || [ -z "$total_count" ] || [ $index -ge "$total_count" ]; then
       log "ERROR: Unable to retrieve any files for ${project_id} from ForgeAPI also Validate files have release type associated with no. ${RELEASE_NUMBER_FILTER}"
       exit 2
     fi
     # Increment start index to new set.
-    index=$(($index + $pageSize))
+    index=$((index + pageSize))
   done
   if [ ! "$PROJECT_FILE" ]; then
     log "ERROR: Unable to retrieve any files for ${project_id}, Release Type: ${RELEASE_NUMBER_FILTER}, FAMILY_TYPE: ${FAMILY,,}"
@@ -157,19 +158,22 @@ downloadModPackfromModFile() {
     log "ERROR: Project File not found from the ForgeAPI"
     exit 2
   fi
-
-  # grabs needed values from our json return
-  unset file_name
-  unset download_url
-  file_name=$(jq -n "$PROJECT_FILE" | jq -jc '.fileName' )
-  download_url=$(jq -n "$PROJECT_FILE" | jq -jc '.downloadUrl' )
-  
   # trys to make the output directory incase it doesnt exist.
   mkdir -p "$out_dir"
-  echo "Downloading ${download_url}"
-  if ! get --skip-up-to-date -o "${out_dir}/${file_name}" "${download_url}"; then
-    log "ERROR: failed to download from ${download_url}"
-    exit 2
+
+  # grabs needed values from our json return
+  file_name=$(jq -n "$PROJECT_FILE" | jq -jc '.fileName // empty' )
+  download_url=$(jq -n "$PROJECT_FILE" | jq -jc '.downloadUrl // empty' )
+  mod_id=$(jq -n "$PROJECT_FILE" | jq -jc '.modId // empty' )
+
+  if [ ! -f "${out_dir}/${file_name}" ]; then
+    echo "Downloading ${download_url}"
+    # Track the mods we have downloaded.
+    DOWNLOADED_MODIDS+=("${mod_id}")
+    if ! get --skip-up-to-date -o "${out_dir}/${file_name}" "${download_url}"; then
+      log "ERROR: failed to download from ${download_url}"
+      exit 2
+    fi
   fi
 }
 
@@ -179,11 +183,15 @@ downloadDependencies(){
     required_dependencies=$(jq -n "$dependencies" | jq --arg REQUIRED_FILTER "3" -jc '
       map(select(.relationType==($REQUIRED_FILTER|tonumber)))')
     if [ "$required_dependencies" ]; then
-      jq -n "$required_dependencies" | jq -c '.[]?' | while read current_dependency; do
+      while read -r current_dependency; do
         mod_id=$(jq -n "$current_dependency" | jq -jc '.modId' )
-        modFileByProjectID $mod_id "release"
-        downloadModPackfromModFile
-      done
+        # Validate we have not tried to download the mod yet.
+        if [[ ! "${DOWNLOADED_MODIDS[*]}" =~ $mod_id ]]; then
+          modFileByProjectID "$mod_id" "release"
+          downloadModPackfromModFile
+        fi
+      # needs to be piped in to keep look in main process
+      done < <(jq -n "$required_dependencies" | jq -c '.[]?')
     fi
   fi
 }
@@ -198,18 +206,22 @@ if [ "$MODS_FORGEAPI_FILE" ] && [ -z "$MODS_FORGEAPI_PROJECTIDS" ]; then
   fi
 
   # Needs loop here to look up release types befor calling download.
-  jq -c '.[]?' $MODS_FORGEAPI_FILE | while read current_project; do
+  while read -r current_project; do
     # Per stack overflow we can use //empty to return empty string that works with -z
     project_id=$(jq -n "$current_project" | jq -r '.projectId // empty' )
     current_release_type=$(jq -n "$current_project" | jq -r '.releaseType // empty' )
     current_file_name=$(jq -n "$current_project" | jq -r '.fileName // empty' )
 
-    modFileByProjectID $project_id $current_release_type $current_file_name
-    downloadModPackfromModFile
-    if isTrue "${MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES}"; then
-      downloadDependencies
+    # Validate we have not tried to download the mod yet.
+    if [[ ! "${DOWNLOADED_MODIDS[*]}" =~ $project_id ]]; then
+      modFileByProjectID "$project_id" "$current_release_type" "$current_file_name"
+      downloadModPackfromModFile
+      if isTrue "${MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES}"; then
+        downloadDependencies
+      fi
     fi
-  done
+  # needs to be piped in to keep look in main process
+  done < <(jq -c '.[]?' $MODS_FORGEAPI_FILE)
 fi
 
 # Use only project ids and global release data.
@@ -217,10 +229,13 @@ if [ "$MODS_FORGEAPI_PROJECTIDS" ] && [ -z "$MODS_FORGEAPI_FILE" ]; then
   ensureModKey
   updateFamilyFilter
   for project_id in ${MODS_FORGEAPI_PROJECTIDS//,/ }; do
-    modFileByProjectID $project_id 
-    downloadModPackfromModFile
-    if isTrue "${MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES}"; then
-      downloadDependencies
+    # Validate we have not tried to download the mod yet.
+    if [[ ! "${DOWNLOADED_MODIDS[*]}" =~ $project_id ]]; then
+      modFileByProjectID $project_id 
+      downloadModPackfromModFile
+      if isTrue "${MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES}"; then
+        downloadDependencies
+      fi
     fi
   done
 fi

tests/setuponlytests/forgeapimods_projectids/verify.sh

@@ -1,4 +1,5 @@
+# No family filter applied, DO NOT use Fabric or Forge specific name validation as it may cause random breakage.
 mc-image-helper assert fileExists "/data/mods/BiomesOPlenty*"
 mc-image-helper assert fileExists "/data/mods/TerraBlender*"
-mc-image-helper assert fileExists "/data/mods/voicechat-fabric*"
+mc-image-helper assert fileExists "/data/mods/voicechat*"
 mc-image-helper assert fileExists "/data/mods/fabric-api*"