Initial Azure Pipelines configuration

.azure-pipelines/README.md

@@ -0,0 +1,3 @@
+## Azure Pipelines Configuration
+
+Please see the [Documentation](https://github.com/ansible/community/wiki/Testing:-Azure-Pipelines) for more information.

.azure-pipelines/azure-pipelines.yml

@@ -0,0 +1,126 @@
+trigger:
+  batch: true
+  branches:
+    include:
+      - main
+      - stable-*
+
+pr:
+  autoCancel: true
+  branches:
+    include:
+      - main
+      - stable-*
+
+schedules:
+  - cron: 0 9 * * *
+    displayName: Nightly
+    always: true
+    branches:
+      include:
+        - main
+        - stable-*
+
+variables:
+  - name: checkoutPath
+    value: ansible_collections/hetzner/hcloud
+  - name: coverageBranches
+    value: main
+  - name: pipelinesCoverage
+    value: coverage
+  - name: entryPoint
+    value: tests/utils/shippable/shippable.sh
+  - name: fetchDepth
+    value: 0
+
+resources:
+  containers:
+    - container: default
+      image: quay.io/ansible/azure-pipelines-test-container:1.7.1
+
+pool: Standard
+
+stages:
+### Sanity 
+  - stage: Ansible_devel
+    displayName: Sanity devel
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          targets:
+            - name: Sanity
+              test: 'devel/sanity/1'
+  - stage: Ansible_2_10
+    displayName: Sanity & Units 2.10
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          targets:
+            - name: Sanity
+              test: '2.10/sanity/1'
+  - stage: Ansible_2_9
+    displayName: Sanity 2.9
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          targets:
+            - name: Sanity
+              test: '2.9/sanity/1'
+
+## Integration tests (remote)
+  - stage: Hetzner_devel
+    displayName: Hetzner devel
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          nameFormat: Python 3.8 {0}
+          testFormat: devel/hcloud/3.8/{0}
+          targets:
+            - test: ''
+          groups:
+            - 1
+            - 2
+  - stage: Hetzner_2_10
+    displayName: Hetzner 2.10
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          nameFormat: Python 3.8 {0}
+          testFormat: 2.10/hcloud/3.8/{0}
+          targets:
+            - test: ''
+          groups:
+            - 1
+            - 2
+            - 3
+  - stage: Hetzner_2_9
+    displayName: Hetzner 2.9
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          nameFormat: Python 3.8 {0}
+          testFormat: 2.9/hcloud/3.8/{0}
+          targets:
+            - test: ''
+          groups:
+            - 1
+            - 2
+            - 3
+### Finally
+  - stage: Summary
+    condition: succeededOrFailed()
+    dependsOn:
+      - Ansible_devel
+      - Ansile_2_10
+      - Ansile_2_9
+      - Hetzner_devel
+      - Hetzner_2_10
+      - Hetzner_2_9
+    jobs:
+      - template: templates/coverage.yml

.azure-pipelines/scripts/aggregate-coverage.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Aggregate code coverage results for later processing.
+
+set -o pipefail -eu
+
+agent_temp_directory="$1"
+
+PATH="${PWD}/bin:${PATH}"
+
+mkdir "${agent_temp_directory}/coverage/"
+
+options=(--venv --venv-system-site-packages --color -v)
+
+ansible-test coverage combine --export "${agent_temp_directory}/coverage/" "${options[@]}"
+
+if ansible-test coverage analyze targets generate --help >/dev/null 2>&1; then
+    # Only analyze coverage if the installed version of ansible-test supports it.
+    # Doing so allows this script to work unmodified for multiple Ansible versions.
+    ansible-test coverage analyze targets generate "${agent_temp_directory}/coverage/coverage-analyze-targets.json" "${options[@]}"
+fi

.azure-pipelines/scripts/combine-coverage.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+"""
+Combine coverage data from multiple jobs, keeping the data only from the most recent attempt from each job.
+Coverage artifacts must be named using the format: "Coverage $(System.JobAttempt) {StableUniqueNameForEachJob}"
+The recommended coverage artifact name format is: Coverage $(System.JobAttempt) $(System.StageDisplayName) $(System.JobDisplayName)
+Keep in mind that Azure Pipelines does not enforce unique job display names (only names).
+It is up to pipeline authors to avoid name collisions when deviating from the recommended format.
+"""
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import re
+import shutil
+import sys
+
+
+def main():
+    """Main program entry point."""
+    source_directory = sys.argv[1]
+
+    if '/ansible_collections/' in os.getcwd():
+        output_path = "tests/output"
+    else:
+        output_path = "test/results"
+
+    destination_directory = os.path.join(output_path, 'coverage')
+
+    if not os.path.exists(destination_directory):
+        os.makedirs(destination_directory)
+
+    jobs = {}
+    count = 0
+
+    for name in os.listdir(source_directory):
+        match = re.search('^Coverage (?P<attempt>[0-9]+) (?P<label>.+)$', name)
+        label = match.group('label')
+        attempt = int(match.group('attempt'))
+        jobs[label] = max(attempt, jobs.get(label, 0))
+
+    for label, attempt in jobs.items():
+        name = 'Coverage {attempt} {label}'.format(label=label, attempt=attempt)
+        source = os.path.join(source_directory, name)
+        source_files = os.listdir(source)
+
+        for source_file in source_files:
+            source_path = os.path.join(source, source_file)
+            destination_path = os.path.join(destination_directory, source_file + '.' + label)
+            print('"%s" -> "%s"' % (source_path, destination_path))
+            shutil.copyfile(source_path, destination_path)
+            count += 1
+
+    print('Coverage file count: %d' % count)
+    print('##vso[task.setVariable variable=coverageFileCount]%d' % count)
+    print('##vso[task.setVariable variable=outputPath]%s' % output_path)
+
+
+if __name__ == '__main__':
+    main()

.azure-pipelines/scripts/process-results.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Check the test results and set variables for use in later steps.
+
+set -o pipefail -eu
+
+if [[ "$PWD" =~ /ansible_collections/ ]]; then
+    output_path="tests/output"
+else
+    output_path="test/results"
+fi
+
+echo "##vso[task.setVariable variable=outputPath]${output_path}"
+
+if compgen -G "${output_path}"'/junit/*.xml' > /dev/null; then
+    echo "##vso[task.setVariable variable=haveTestResults]true"
+fi
+
+if compgen -G "${output_path}"'/bot/ansible-test-*' > /dev/null; then
+    echo "##vso[task.setVariable variable=haveBotResults]true"
+fi
+
+if compgen -G "${output_path}"'/coverage/*' > /dev/null; then
+    echo "##vso[task.setVariable variable=haveCoverageData]true"
+fi

.azure-pipelines/scripts/publish-codecov.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Upload code coverage reports to codecov.io.
+# Multiple coverage files from multiple languages are accepted and aggregated after upload.
+# Python coverage, as well as PowerShell and Python stubs can all be uploaded.
+
+set -o pipefail -eu
+
+output_path="$1"
+
+curl --silent --show-error https://codecov.io/bash > codecov.sh
+
+for file in "${output_path}"/reports/coverage*.xml; do
+    name="${file}"
+    name="${name##*/}"  # remove path
+    name="${name##coverage=}"  # remove 'coverage=' prefix if present
+    name="${name%.xml}"  # remove '.xml' suffix
+
+    bash codecov.sh \
+        -f "${file}" \
+        -n "${name}" \
+        -X coveragepy \
+        -X gcov \
+        -X fix \
+        -X search \
+        -X xcode \
+        || echo "Failed to upload code coverage report to codecov.io: ${file}"
+done

.azure-pipelines/scripts/report-coverage.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Generate code coverage reports for uploading to Azure Pipelines and codecov.io.
+
+set -o pipefail -eu
+
+PATH="${PWD}/bin:${PATH}"
+
+if ! ansible-test --help >/dev/null 2>&1; then
+    # Install the devel version of ansible-test for generating code coverage reports.
+    # This is only used by Ansible Collections, which are typically tested against multiple Ansible versions (in separate jobs).
+    # Since a version of ansible-test is required that can work the output from multiple older releases, the devel version is used.
+    pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check
+fi
+
+ansible-test coverage xml --stub --venv --venv-system-site-packages --color -v

.azure-pipelines/scripts/run-tests.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Configure the test environment and run the tests.
+
+set -o pipefail -eu
+
+entry_point="$1"
+test="$2"
+read -r -a coverage_branches <<< "$3"  # space separated list of branches to run code coverage on for scheduled builds
+
+export COMMIT_MESSAGE
+export COMPLETE
+export COVERAGE
+export IS_PULL_REQUEST
+
+if [ "${SYSTEM_PULLREQUEST_TARGETBRANCH:-}" ]; then
+    IS_PULL_REQUEST=true
+    COMMIT_MESSAGE=$(git log --format=%B -n 1 HEAD^2)
+else
+    IS_PULL_REQUEST=
+    COMMIT_MESSAGE=$(git log --format=%B -n 1 HEAD)
+fi
+
+COMPLETE=
+COVERAGE=
+
+if [ "${BUILD_REASON}" = "Schedule" ]; then
+    COMPLETE=yes
+
+    if printf '%s\n' "${coverage_branches[@]}" | grep -q "^${BUILD_SOURCEBRANCHNAME}$"; then
+        COVERAGE=yes
+    fi
+fi
+
+"${entry_point}" "${test}" 2>&1 | "$(dirname "$0")/time-command.py"

.azure-pipelines/scripts/time-command.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+"""Prepends a relative timestamp to each input line from stdin and writes it to stdout."""
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import sys
+import time
+
+
+def main():
+    """Main program entry point."""
+    start = time.time()
+
+    sys.stdin.reconfigure(errors='surrogateescape')
+    sys.stdout.reconfigure(errors='surrogateescape')
+
+    for line in sys.stdin:
+        seconds = time.time() - start
+        sys.stdout.write('%02d:%02d %s' % (seconds // 60, seconds % 60, line))
+        sys.stdout.flush()
+
+
+if __name__ == '__main__':
+    main()

.azure-pipelines/templates/coverage.yml

@@ -0,0 +1,39 @@
+# This template adds a job for processing code coverage data.
+# It will upload results to Azure Pipelines and codecov.io.
+# Use it from a job stage that completes after all other jobs have completed.
+# This can be done by placing it in a separate summary stage that runs after the test stage(s) have completed.
+
+jobs:
+  - job: Coverage
+    displayName: Code Coverage
+    container: default
+    workspace:
+      clean: all
+    steps:
+      - checkout: self
+        fetchDepth: $(fetchDepth)
+        path: $(checkoutPath)
+      - task: DownloadPipelineArtifact@2
+        displayName: Download Coverage Data
+        inputs:
+          path: coverage/
+          patterns: "Coverage */*=coverage.combined"
+      - bash: .azure-pipelines/scripts/combine-coverage.py coverage/
+        displayName: Combine Coverage Data
+      - bash: .azure-pipelines/scripts/report-coverage.sh
+        displayName: Generate Coverage Report
+        condition: gt(variables.coverageFileCount, 0)
+      - task: PublishCodeCoverageResults@1
+        inputs:
+          codeCoverageTool: Cobertura
+          # Azure Pipelines only accepts a single coverage data file.
+          # That means only Python or PowerShell coverage can be uploaded, but not both.
+          # Set the "pipelinesCoverage" variable to determine which type is uploaded.
+          # Use "coverage" for Python and "coverage-powershell" for PowerShell.
+          summaryFileLocation: "$(outputPath)/reports/$(pipelinesCoverage).xml"
+        displayName: Publish to Azure Pipelines
+        condition: gt(variables.coverageFileCount, 0)
+      - bash: .azure-pipelines/scripts/publish-codecov.sh "$(outputPath)"
+        displayName: Publish to codecov.io
+        condition: gt(variables.coverageFileCount, 0)
+        continueOnError: true

.azure-pipelines/templates/matrix.yml

@@ -0,0 +1,55 @@
+# This template uses the provided targets and optional groups to generate a matrix which is then passed to the test template.
+# If this matrix template does not provide the required functionality, consider using the test template directly instead.
+
+parameters:
+  # A required list of dictionaries, one per test target.
+  # Each item in the list must contain a "test" or "name" key.
+  # Both may be provided. If one is omitted, the other will be used.
+  - name: targets
+    type: object
+
+  # An optional list of values which will be used to multiply the targets list into a matrix.
+  # Values can be strings or numbers.
+  - name: groups
+    type: object
+    default: []
+
+  # An optional format string used to generate the job name.
+  # - {0} is the name of an item in the targets list.
+  - name: nameFormat
+    type: string
+    default: "{0}"
+
+  # An optional format string used to generate the test name.
+  # - {0} is the name of an item in the targets list.
+  - name: testFormat
+    type: string
+    default: "{0}"
+
+  # An optional format string used to add the group to the job name.
+  # {0} is the formatted name of an item in the targets list.
+  # {{1}} is the group -- be sure to include the double "{{" and "}}".
+  - name: nameGroupFormat
+    type: string
+    default: "{0} - {{1}}"
+
+  # An optional format string used to add the group to the test name.
+  # {0} is the formatted test of an item in the targets list.
+  # {{1}} is the group -- be sure to include the double "{{" and "}}".
+  - name: testGroupFormat
+    type: string
+    default: "{0}/{{1}}"
+
+jobs:
+  - template: test.yml
+    parameters:
+      jobs:
+        - ${{ if eq(length(parameters.groups), 0) }}:
+          - ${{ each target in parameters.targets }}:
+            - name: ${{ format(parameters.nameFormat, coalesce(target.name, target.test)) }}
+              test: ${{ format(parameters.testFormat, coalesce(target.test, target.name)) }}
+        - ${{ if not(eq(length(parameters.groups), 0)) }}:
+          - ${{ each group in parameters.groups }}:
+            - ${{ each target in parameters.targets }}:
+              - name: ${{ format(format(parameters.nameGroupFormat, parameters.nameFormat), coalesce(target.name, target.test), group) }}
+                test: ${{ format(format(parameters.testGroupFormat, parameters.testFormat), coalesce(target.test, target.name), group) }}

.azure-pipelines/templates/test.yml

@@ -0,0 +1,45 @@
+# This template uses the provided list of jobs to create test one or more test jobs.
+# It can be used directly if needed, or through the matrix template.
+
+parameters:
+  # A required list of dictionaries, one per test job.
+  # Each item in the list must contain a "job" and "name" key.
+  - name: jobs
+    type: object
+
+jobs:
+  - ${{ each job in parameters.jobs }}:
+    - job: test_${{ replace(replace(replace(job.test, '/', '_'), '.', '_'), '-', '_') }}
+      displayName: ${{ job.name }}
+      container: default
+      workspace:
+        clean: all
+      steps:
+        - checkout: self
+          fetchDepth: $(fetchDepth)
+          path: $(checkoutPath)
+        - bash: .azure-pipelines/scripts/run-tests.sh "$(entryPoint)" "${{ job.test }}" "$(coverageBranches)"
+          displayName: Run Tests
+        - bash: .azure-pipelines/scripts/process-results.sh
+          condition: succeededOrFailed()
+          displayName: Process Results
+        - bash: .azure-pipelines/scripts/aggregate-coverage.sh "$(Agent.TempDirectory)"
+          condition: eq(variables.haveCoverageData, 'true')
+          displayName: Aggregate Coverage Data
+        - task: PublishTestResults@2
+          condition: eq(variables.haveTestResults, 'true')
+          inputs:
+            testResultsFiles: "$(outputPath)/junit/*.xml"
+          displayName: Publish Test Results
+        - task: PublishPipelineArtifact@1
+          condition: eq(variables.haveBotResults, 'true')
+          displayName: Publish Bot Results
+          inputs:
+            targetPath: "$(outputPath)/bot/"
+            artifactName: "Bot $(System.JobAttempt) $(System.StageDisplayName) $(System.JobDisplayName)"
+        - task: PublishPipelineArtifact@1
+          condition: eq(variables.haveCoverageData, 'true')
+          displayName: Publish Coverage Data
+          inputs:
+            targetPath: "$(Agent.TempDirectory)/coverage/"
+            artifactName: "Coverage $(System.JobAttempt) $(System.StageDisplayName) $(System.JobDisplayName)"

shippable.yml

@@ -1,56 +0,0 @@
-language: python
-
-python: 3.7
-env:
-  matrix:
-    - T=none
-
-matrix:
-  exclude:
-    - env: T=none
-  include:
-    - env: T=devel/sanity/1
-      python: 3.7
-    - env: T=devel/sanity/2
-      python: 3.7
-    - env: T=devel/sanity/3
-      python: 3.7
-    - env: T=devel/sanity/4
-      python: 3.7
-
-    - env: T=devel/hcloud/3.8/1
-    - env: T=devel/hcloud/3.8/2
-
-
-    - env: T=2.10/sanity/1
-    - env: T=2.10/sanity/2
-    - env: T=2.10/sanity/3
-    - env: T=2.10/sanity/4
-
-    - env: T=2.10/hcloud/3.8/1
-    - env: T=2.10/hcloud/3.8/2
-    - env: T=2.10/hcloud/3.8/3
-
-
-    - env: T=2.9/sanity/1
-    - env: T=2.9/sanity/2
-    - env: T=2.9/sanity/3
-    - env: T=2.9/sanity/4
-
-branches:
-  except:
-    - "*-patch-*"
-    - "revert-*-*"
-
-build:
-  ci:
-    - tests/utils/shippable/timing.sh tests/utils/shippable/shippable.sh $T
-
-integrations:
-  notifications:
-    - integrationName: email
-      type: email
-      on_success: never
-      on_failure: never
-      on_start: never
-      on_pull_request: never

tests/utils/shippable/sanity.sh

@@ -14,31 +14,14 @@ else
 fi
 
 if [ "${group}" == "extra" ]; then
-    pip install antsibull-changelog
-    python ../../community/internal_test_tools/tools/run.py --color
+    # ansible-galaxy -vvv collection install community.internal_test_tools
+    git clone --single-branch --depth 1 https://github.com/ansible-collections/community.internal_test_tools.git ../../community/internal_test_tools
+
+    ../internal_test_tools/tools/run.py --color
     exit
 fi
 
-case "${group}" in
-    1) options=(--skip-test pylint --skip-test ansible-doc --skip-test validate-modules) ;;
-    2) options=(                   --test      ansible-doc      --test validate-modules) ;;
-    3) options=(--test pylint           plugins/modules/) ;;
-    4) options=(--test pylint --exclude plugins/modules/) ;;
-esac
-
-# allow collection migration sanity tests for groups 3 and 4 to pass without updating this script during migration
-network_path="lib/ansible/modules/network/"
-
-if [ -d "${network_path}" ]; then
-    if [ "${group}" -eq 3 ]; then
-        options+=(--exclude "${network_path}")
-    elif [ "${group}" -eq 4 ]; then
-        options+=("${network_path}")
-    fi
-fi
-
 # shellcheck disable=SC2086
 ansible-test sanity --color -v --junit ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
     --docker --base-branch "${base_branch}" \
-    --exclude shippable.yml --exclude tests/utils/ \
-    "${options[@]}" --allow-disabled
+    --allow-disabled

tests/utils/shippable/shippable.sh

@@ -14,13 +14,15 @@ function join {
     echo "$*";
 }
 
-test="$(join / "${args[@]:1}")"
+# Ensure we can write other collections to this dir
+sudo chown "$(whoami)" "${PWD}/../../"
 
+test="$(join / "${args[@]:1}")"
 docker images ansible/ansible
 docker images quay.io/ansible/*
 docker ps
 
-for container in $(docker ps --format '{{.Image}} {{.ID}}' | grep -v '^drydock/' | sed 's/^.* //'); do
+for container in $(docker ps --format '{{.Image}} {{.ID}}' | grep -v -e '^drydock/' -e '^quay.io/ansible/azure-pipelines-test-container:' | sed 's/^.* //'); do
     docker rm -f "${container}" || true  # ignore errors
 done
 
@@ -35,6 +37,7 @@ python -V
 
 function retry
 {
+    # shellcheck disable=SC2034
     for repetition in 1 2 3; do
         set +e
         "$@"
@@ -43,7 +46,7 @@ function retry
         if [ ${result} == 0 ]; then
             return ${result}
         fi
-        echo "$@ -> ${result}"
+        echo "@* -> ${result}"
     done
     echo "Command '$@' failed 3 times!"
     exit -1
@@ -57,12 +60,16 @@ if [ "${ansible_version}" == "devel" ]; then
 else
     retry pip install "https://github.com/ansible/ansible/archive/stable-${ansible_version}.tar.gz" --disable-pip-version-check
 fi
-export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible"
-SHIPPABLE_RESULT_DIR="$(pwd)/shippable"
-TEST_DIR="${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/hetzner/hcloud"
-mkdir -p "${TEST_DIR}"
-cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}"
-cd "${TEST_DIR}"
+if [ "${SHIPPABLE_BUILD_ID:-}" ]; then
+    export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible"
+    SHIPPABLE_RESULT_DIR="$(pwd)/shippable"
+    TEST_DIR="${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/hetzner/hcloud"
+    mkdir -p "${TEST_DIR}"
+    cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}"
+    cd "${TEST_DIR}"
+else
+    export ANSIBLE_COLLECTIONS_PATHS="${PWD}/../../../"
+fi
 
 # STAR: HACK install dependencies
 retry ansible-galaxy -vvv collection install community.general
@@ -117,9 +124,9 @@ function cleanup
     fi
 }
 
-trap cleanup EXIT
+if [ "${SHIPPABLE_BUILD_ID:-}" ]; then trap cleanup EXIT; fi
 
 ansible-test env --dump --show --timeout "50" --color -v
 
-"tests/utils/shippable/check_matrix.py"
+if [ "${SHIPPABLE_BUILD_ID:-}" ]; then "tests/utils/shippable/check_matrix.py"; fi
 "tests/utils/shippable/${script}.sh" "${test}"