AZP fixes (#52)

* Tidy up files
* Add Sanity ignore files
* Use master as branch name instead of main
* Use targets in integration jobs
* Remove tests/sanity/requirements.txt (not used by ansible-test)
* sanity: prevent import exception for inventory script

Co-authored-by: David Moreau Simard <moi@dmsimard.com>

.azure-pipelines/azure-pipelines.yml

@@ -2,15 +2,13 @@ trigger:
   batch: true
   branches:
     include:
-      - main
-      - stable-*
+      - master
 
 pr:
   autoCancel: true
   branches:
     include:
-      - main
-      - stable-*
+      - master
 
 schedules:
   - cron: 0 9 * * *
@@ -18,14 +16,13 @@ schedules:
     always: true
     branches:
       include:
-        - main
-        - stable-*
+        - master
 
 variables:
   - name: checkoutPath
     value: ansible_collections/hetzner/hcloud
   - name: coverageBranches
-    value: main
+    value: master
   - name: pipelinesCoverage
     value: coverage
   - name: entryPoint
@@ -41,7 +38,7 @@ resources:
 pool: Standard
 
 stages:
-### Sanity 
+### Sanity
   - stage: Ansible_devel
     displayName: Sanity devel
     dependsOn: []
@@ -77,41 +74,41 @@ stages:
     jobs:
       - template: templates/matrix.yml
         parameters:
-          nameFormat: Python 3.8 {0}
-          testFormat: devel/hcloud/3.8/{0}
-          targets:
-            - test: ''
           groups:
             - 1
             - 2
+          targets:
+            - name: hcloud
+              test: 'devel/hcloud/3.8'
+
   - stage: Hetzner_2_10
     displayName: Hetzner 2.10
     dependsOn: []
     jobs:
       - template: templates/matrix.yml
         parameters:
-          nameFormat: Python 3.8 {0}
-          testFormat: 2.10/hcloud/3.8/{0}
-          targets:
-            - test: ''
           groups:
             - 1
             - 2
             - 3
+          targets:
+            - name: hcloud
+              test: '2.10/hcloud/3.8'
+
   - stage: Hetzner_2_9
     displayName: Hetzner 2.9
     dependsOn: []
     jobs:
       - template: templates/matrix.yml
         parameters:
-          nameFormat: Python 3.8 {0}
-          testFormat: 2.9/hcloud/3.8/{0}
-          targets:
-            - test: ''
           groups:
             - 1
             - 2
             - 3
+          targets:
+            - name: hcloud
+              test: '2.9/hcloud/3.8'
+
 ### Finally
   - stage: Summary
     condition: succeededOrFailed()

plugins/inventory/hcloud.py

@@ -102,8 +102,9 @@ from ansible.release import __version__
 try:
     from hcloud import hcloud
     from hcloud import APIException
+    HAS_HCLOUD = True
 except ImportError:
-    raise AnsibleError("The Hetzner Cloud dynamic inventory plugin requires hcloud-python.")
+    HAS_HCLOUD = False
 
 
 class InventoryModule(BaseInventoryPlugin, Constructable):
@@ -243,6 +244,10 @@ class InventoryModule(BaseInventoryPlugin, Constructable):
 
     def parse(self, inventory, loader, path, cache=True):
         super(InventoryModule, self).parse(inventory, loader, path, cache)
+
+        if not HAS_HCLOUD:
+            raise AnsibleError("The Hetzner Cloud dynamic inventory plugin requires hcloud-python.")
+
         self._read_config_data(path)
         self._configure_hcloud_client()
         self._test_hcloud_token()

tests/sanity/ignore-2.10.txt

@@ -0,0 +1,2 @@
+tests/utils/shippable/check_matrix.py replace-urlopen
+tests/utils/shippable/timing.py shebang

tests/sanity/ignore-2.11.txt

@@ -0,0 +1,2 @@
+tests/utils/shippable/check_matrix.py replace-urlopen
+tests/utils/shippable/timing.py shebang

tests/sanity/ignore-2.9.txt

@@ -0,0 +1,2 @@
+tests/utils/shippable/check_matrix.py replace-urlopen
+tests/utils/shippable/timing.py shebang

tests/sanity/requirements.txt

@@ -1,4 +0,0 @@
-packaging  # needed for update-bundled and changelog
-sphinx ; python_version >= '3.5' # docs build requires python 3+
-sphinx-notfound-page ; python_version >= '3.5' # docs build requires python 3+
-straight.plugin ; python_version >= '3.5' # needed for hacking/build-ansible.py which will host changelog generation and requires python 3+

tests/utils/gitlab/gitlab.sh

@@ -7,6 +7,7 @@ declare -a args
 IFS='/:' read -ra args <<< "$1"
 
 ansible_version="${args[0]}"
+# shellcheck disable=SC2034
 script="${args[1]}"
 
 function join {
@@ -21,6 +22,7 @@ python -V
 
 function retry
 {
+    # shellcheck disable=SC2034
     for repetition in 1 2 3; do
         set +e
         "$@"
@@ -29,9 +31,9 @@ function retry
         if [ ${result} == 0 ]; then
             return ${result}
         fi
-        echo "$@ -> ${result}"
+        echo "@* -> ${result}"
     done
-    echo "Command '$@' failed 3 times!"
+    echo "Command '@*' failed 3 times!"
     exit -1
 }
 
@@ -44,6 +46,7 @@ else
     retry pip install "https://github.com/ansible/ansible/archive/stable-${ansible_version}.tar.gz" --disable-pip-version-check
 fi
 export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible"
+# shellcheck disable=SC2034
 SHIPPABLE_RESULT_DIR="$(pwd)/shippable"
 TEST_DIR="${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/hetzner/hcloud"
 rm -rf  "${TEST_DIR}"
@@ -87,7 +90,7 @@ find plugins -type d -empty -print -delete
 ansible-test env --dump --show --timeout "50" --color -v
 
 group="${args[1]}"
-echo $test
+echo "$test"
 if [[ "${test}" =~ hcloud ]]; then
   group="${args[3]}"
   bash tests/utils/gitlab/integration.sh "shippable/hcloud/group${group}/"

tests/utils/gitlab/integration.sh

@@ -3,13 +3,18 @@
 target="$1"
 
 HCLOUD_TOKEN=$(cat hcloud_token.txt)
+# shellcheck disable=SC2034,SC2154
 changed_all_target="shippable/${cloud}/smoketest/"
 
+# shellcheck disable=SC2046
 echo "[default]
 hcloud_api_token=${HCLOUD_TOKEN}
 " >> $(pwd)/tests/integration/cloud-config-hcloud.ini
-# shellcheck disable=SC2086
 export SHIPPABLE="true"
+
+# shellcheck disable=SC2155
 export SHIPPABLE_BUILD_NUMBER="gl-$(cat prefix.txt)"
+
+# shellcheck disable=SC2155,SC2002
 export SHIPPABLE_JOB_NUMBER="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 2 | head -n 1)"
 ansible-test integration --color --local -vv "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"}

tests/utils/shippable/shippable.sh

@@ -18,6 +18,7 @@ function join {
 sudo chown "$(whoami)" "${PWD}/../../"
 
 test="$(join / "${args[@]:1}")"
+
 docker images ansible/ansible
 docker images quay.io/ansible/*
 docker ps
@@ -48,8 +49,8 @@ function retry
         fi
         echo "@* -> ${result}"
     done
-    echo "Command '$@' failed 3 times!"
-    exit -1
+    echo "Command '@*' failed 3 times!"
+    exit 1
 }
 
 command -v pip
@@ -60,6 +61,7 @@ if [ "${ansible_version}" == "devel" ]; then
 else
     retry pip install "https://github.com/ansible/ansible/archive/stable-${ansible_version}.tar.gz" --disable-pip-version-check
 fi
+
 if [ "${SHIPPABLE_BUILD_ID:-}" ]; then
     export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible"
     SHIPPABLE_RESULT_DIR="$(pwd)/shippable"
@@ -83,9 +85,20 @@ retry ansible-galaxy -vvv collection install community.internal_test_tools
 export PYTHONIOENCODING='utf-8'
 
 if [ "${JOB_TRIGGERED_BY_NAME:-}" == "nightly-trigger" ]; then
+    COVERAGE=yes
     COMPLETE=yes
 fi
 
+if [ -n "${COVERAGE:-}" ]; then
+    # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value
+    export COVERAGE="--coverage"
+elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then
+    # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message
+    export COVERAGE="--coverage"
+else
+    # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled)
+    export COVERAGE="--coverage-check"
+fi
 
 if [ -n "${COMPLETE:-}" ]; then
     # disable change detection triggered by setting the COMPLETE environment variable to a non-empty value
@@ -111,6 +124,68 @@ find plugins -type d -empty -print -delete
 
 function cleanup
 {
+    # for complete on-demand coverage generate a report for all files with no coverage on the "sanity/5" job so we only have one copy
+    if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/5" ]; then
+        stub="--stub"
+        # trigger coverage reporting for stubs even if no other coverage data exists
+        mkdir -p tests/output/coverage/
+    else
+        stub=""
+    fi
+
+    if [ -d tests/output/coverage/ ]; then
+        if find tests/output/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then
+            process_coverage='yes'  # process existing coverage files
+        elif [ "${stub}" ]; then
+            process_coverage='yes'  # process coverage when stubs are enabled
+        else
+            process_coverage=''
+        fi
+
+        if [ "${process_coverage}" ]; then
+            # use python 3.7 for coverage to avoid running out of memory during coverage xml processing
+            # only use it for coverage to avoid the additional overhead of setting up a virtual environment for a potential no-op job
+            virtualenv --python /usr/bin/python3.7 ~/ansible-venv
+            set +ux
+            . ~/ansible-venv/bin/activate
+            set -ux
+
+            # shellcheck disable=SC2086
+            ansible-test coverage xml --color -v --requirements --group-by command --group-by version ${stub:+"$stub"}
+            cp -a tests/output/reports/coverage=*.xml "$SHIPPABLE_RESULT_DIR/codecoverage/"
+
+            if [ "${ansible_version}" != "2.9" ]; then
+                # analyze and capture code coverage aggregated by integration test target
+                ansible-test coverage analyze targets generate -v "$SHIPPABLE_RESULT_DIR/testresults/coverage-analyze-targets.json"
+            fi
+
+            # upload coverage report to codecov.io only when using complete on-demand coverage
+            if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then
+                for file in tests/output/reports/coverage=*.xml; do
+                    flags="${file##*/coverage=}"
+                    flags="${flags%-powershell.xml}"
+                    flags="${flags%.xml}"
+                    # remove numbered component from stub files when converting to tags
+                    flags="${flags//stub-[0-9]*/stub}"
+                    flags="${flags//=/,}"
+                    flags="${flags//[^a-zA-Z0-9_,]/_}"
+
+                    bash <(curl -s https://codecov.io/bash) \
+                        -f "${file}" \
+                        -F "${flags}" \
+                        -n "${test}" \
+                        -t 8a86e979-f37b-4d5d-95a4-960c280d5eaa \
+                        -X coveragepy \
+                        -X gcov \
+                        -X fix \
+                        -X search \
+                        -X xcode \
+                    || echo "Failed to upload code coverage report to codecov.io: ${file}"
+                done
+            fi
+        fi
+    fi
+
     if [ -d  tests/output/junit/ ]; then
       cp -aT tests/output/junit/ "$SHIPPABLE_RESULT_DIR/testresults/"
     fi
@@ -126,7 +201,13 @@ function cleanup
 
 if [ "${SHIPPABLE_BUILD_ID:-}" ]; then trap cleanup EXIT; fi
 
-ansible-test env --dump --show --timeout "50" --color -v
+if [[ "${COVERAGE:-}" == "--coverage" ]]; then
+    timeout=60
+else
+    timeout=45
+fi
+
+ansible-test env --dump --show --timeout "${timeout}" --color -v
 
 if [ "${SHIPPABLE_BUILD_ID:-}" ]; then "tests/utils/shippable/check_matrix.py"; fi
 "tests/utils/shippable/${script}.sh" "${test}"