mirror of
https://github.com/uutils/coreutils
synced 2024-12-13 23:02:38 +00:00
406 lines
17 KiB
YAML
406 lines
17 KiB
YAML
name: GnuTests
|
|
|
|
# spell-checker:ignore (abbrev/names) CodeCov gnulib GnuTests Swatinem
|
|
# spell-checker:ignore (jargon) submodules
|
|
# spell-checker:ignore (libs/utils) autopoint chksum gperf lcov libexpect pyinotify shopt texinfo valgrind libattr libcap taiki-e
|
|
# spell-checker:ignore (options) Ccodegen Coverflow Cpanic Zpanic
|
|
# spell-checker:ignore (people) Dawid Dziurla * dawidd dtolnay
|
|
# spell-checker:ignore (vars) FILESET SUBDIRS XPASS
|
|
|
|
# * note: to run a single test => `REPO/util/run-gnu-test.sh PATH/TO/TEST/SCRIPT`
|
|
|
|
on:
|
|
pull_request:
|
|
push:
|
|
branches:
|
|
- main
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# End the current execution if there is a new changeset in the PR.
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
|
|
|
|
jobs:
|
|
gnu:
|
|
permissions:
|
|
actions: read # for dawidd6/action-download-artifact to query and download artifacts
|
|
contents: read # for actions/checkout to fetch code
|
|
pull-requests: read # for dawidd6/action-download-artifact to query commit hash
|
|
name: Run GNU tests
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
- name: Initialize workflow variables
|
|
id: vars
|
|
shell: bash
|
|
run: |
|
|
## VARs setup
|
|
outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
|
|
# * config
|
|
path_GNU="gnu"
|
|
path_GNU_tests="${path_GNU}/tests"
|
|
path_UUTILS="uutils"
|
|
path_reference="reference"
|
|
outputs path_GNU path_GNU_tests path_reference path_UUTILS
|
|
#
|
|
repo_default_branch="${{ github.event.repository.default_branch }}"
|
|
repo_GNU_ref="v9.5"
|
|
repo_reference_branch="${{ github.event.repository.default_branch }}"
|
|
outputs repo_default_branch repo_GNU_ref repo_reference_branch
|
|
#
|
|
SUITE_LOG_FILE="${path_GNU_tests}/test-suite.log"
|
|
ROOT_SUITE_LOG_FILE="${path_GNU_tests}/test-suite-root.log"
|
|
TEST_LOGS_GLOB="${path_GNU_tests}/**/*.log" ## note: not usable at bash CLI; [why] double globstar not enabled by default b/c MacOS includes only bash v3 which doesn't have double globstar support
|
|
TEST_FILESET_PREFIX='test-fileset-IDs.sha1#'
|
|
TEST_FILESET_SUFFIX='.txt'
|
|
TEST_SUMMARY_FILE='gnu-result.json'
|
|
TEST_FULL_SUMMARY_FILE='gnu-full-result.json'
|
|
outputs SUITE_LOG_FILE ROOT_SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE
|
|
- name: Checkout code (uutil)
|
|
uses: actions/checkout@v4
|
|
with:
|
|
path: '${{ steps.vars.outputs.path_UUTILS }}'
|
|
- uses: dtolnay/rust-toolchain@master
|
|
with:
|
|
toolchain: stable
|
|
components: rustfmt
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
workspaces: "./${{ steps.vars.outputs.path_UUTILS }} -> target"
|
|
- name: Checkout code (GNU coreutils)
|
|
uses: actions/checkout@v4
|
|
with:
|
|
repository: 'coreutils/coreutils'
|
|
path: '${{ steps.vars.outputs.path_GNU }}'
|
|
ref: ${{ steps.vars.outputs.repo_GNU_ref }}
|
|
submodules: recursive
|
|
- name: Retrieve reference artifacts
|
|
uses: dawidd6/action-download-artifact@v5
|
|
# ref: <https://github.com/dawidd6/action-download-artifact>
|
|
continue-on-error: true ## don't break the build for missing reference artifacts (may be expired or just not generated yet)
|
|
with:
|
|
workflow: GnuTests.yml
|
|
branch: "${{ steps.vars.outputs.repo_reference_branch }}"
|
|
# workflow_conclusion: success ## (default); * but, if commit with failed GnuTests is merged into the default branch, future commits will all show regression errors in GnuTests CI until o/w fixed
|
|
workflow_conclusion: completed ## continually recalibrates to last commit of default branch with a successful GnuTests (ie, "self-heals" from GnuTest regressions, but needs more supervision for/of regressions)
|
|
path: "${{ steps.vars.outputs.path_reference }}"
|
|
- name: Install dependencies
|
|
shell: bash
|
|
run: |
|
|
## Install dependencies
|
|
sudo apt-get update
|
|
sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev
|
|
- name: Add various locales
|
|
shell: bash
|
|
run: |
|
|
## Add various locales
|
|
echo "Before:"
|
|
locale -a
|
|
## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory'
|
|
## Some others need a French locale
|
|
sudo locale-gen
|
|
sudo locale-gen --keep-existing fr_FR
|
|
sudo locale-gen --keep-existing fr_FR.UTF-8
|
|
sudo locale-gen --keep-existing sv_SE
|
|
sudo locale-gen --keep-existing sv_SE.UTF-8
|
|
sudo locale-gen --keep-existing en_US
|
|
sudo locale-gen --keep-existing ru_RU.KOI8-R
|
|
|
|
sudo update-locale
|
|
echo "After:"
|
|
locale -a
|
|
- name: Build binaries
|
|
shell: bash
|
|
run: |
|
|
## Build binaries
|
|
cd '${{ steps.vars.outputs.path_UUTILS }}'
|
|
bash util/build-gnu.sh --release-build
|
|
- name: Run GNU tests
|
|
shell: bash
|
|
run: |
|
|
## Run GNU tests
|
|
path_GNU='${{ steps.vars.outputs.path_GNU }}'
|
|
path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
|
|
bash "${path_UUTILS}/util/run-gnu-test.sh"
|
|
- name: Run GNU root tests
|
|
shell: bash
|
|
run: |
|
|
## Run GNU root tests
|
|
path_GNU='${{ steps.vars.outputs.path_GNU }}'
|
|
path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
|
|
bash "${path_UUTILS}/util/run-gnu-test.sh" run-root
|
|
- name: Extract testing info into JSON
|
|
shell: bash
|
|
run : |
|
|
## Extract testing info into JSON
|
|
path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
|
|
python ${path_UUTILS}/util/gnu-json-result.py ${{ steps.vars.outputs.path_GNU_tests }} > ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }}
|
|
- name: Extract/summarize testing info
|
|
id: summary
|
|
shell: bash
|
|
run: |
|
|
## Extract/summarize testing info
|
|
outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
|
|
#
|
|
path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
|
|
#
|
|
SUITE_LOG_FILE='${{ steps.vars.outputs.SUITE_LOG_FILE }}'
|
|
ROOT_SUITE_LOG_FILE='${{ steps.vars.outputs.ROOT_SUITE_LOG_FILE }}'
|
|
ls -al ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE}
|
|
|
|
if test -f "${SUITE_LOG_FILE}"
|
|
then
|
|
source ${path_UUTILS}/util/analyze-gnu-results.sh ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE}
|
|
if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then
|
|
echo "::error ::Failed to parse test results from '${SUITE_LOG_FILE}'; failing early"
|
|
exit 1
|
|
fi
|
|
output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR / SKIP: $SKIP"
|
|
echo "${output}"
|
|
if [[ "$FAIL" -gt 0 || "$ERROR" -gt 0 ]]; then echo "::warning ::${output}" ; fi
|
|
jq -n \
|
|
--arg date "$(date --rfc-email)" \
|
|
--arg sha "$GITHUB_SHA" \
|
|
--arg total "$TOTAL" \
|
|
--arg pass "$PASS" \
|
|
--arg skip "$SKIP" \
|
|
--arg fail "$FAIL" \
|
|
--arg xpass "$XPASS" \
|
|
--arg error "$ERROR" \
|
|
'{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}'
|
|
HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1)
|
|
outputs HASH
|
|
else
|
|
echo "::error ::Failed to find summary of test results (missing '${SUITE_LOG_FILE}'); failing early"
|
|
exit 1
|
|
fi
|
|
# Compress logs before upload (fails otherwise)
|
|
gzip ${{ steps.vars.outputs.TEST_LOGS_GLOB }}
|
|
- name: Reserve SHA1/ID of 'test-summary'
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: "${{ steps.summary.outputs.HASH }}"
|
|
path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
|
|
- name: Reserve test results summary
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: test-summary
|
|
path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
|
|
- name: Reserve test logs
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: test-logs
|
|
path: "${{ steps.vars.outputs.TEST_LOGS_GLOB }}"
|
|
- name: Upload full json results
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: gnu-full-result.json
|
|
path: ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }}
|
|
- name: Compare test failures VS reference
|
|
shell: bash
|
|
run: |
|
|
## Compare test failures VS reference
|
|
have_new_failures=""
|
|
REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log'
|
|
ROOT_REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite-root.log'
|
|
REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json'
|
|
REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}'
|
|
path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
|
|
# https://github.com/uutils/coreutils/issues/4294
|
|
# https://github.com/uutils/coreutils/issues/4295
|
|
IGNORE_INTERMITTENT="${path_UUTILS}/.github/workflows/ignore-intermittent.txt"
|
|
|
|
mkdir -p ${{ steps.vars.outputs.path_reference }}
|
|
|
|
COMMENT_DIR="${{ steps.vars.outputs.path_reference }}/comment"
|
|
mkdir -p ${COMMENT_DIR}
|
|
echo ${{ github.event.number }} > ${COMMENT_DIR}/NR
|
|
COMMENT_LOG="${COMMENT_DIR}/result.txt"
|
|
|
|
# The comment log might be downloaded from a previous run
|
|
# We only want the new changes, so remove it if it exists.
|
|
rm -f ${COMMENT_LOG}
|
|
touch ${COMMENT_LOG}
|
|
|
|
compare_tests() {
|
|
local new_log_file=$1
|
|
local ref_log_file=$2
|
|
local test_type=$3 # "standard" or "root"
|
|
|
|
if test -f "${ref_log_file}"; then
|
|
echo "Reference ${test_type} test log SHA1/ID: $(sha1sum -- "${ref_log_file}") - ${test_type}"
|
|
REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort)
|
|
CURRENT_RUN_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort)
|
|
REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort)
|
|
CURRENT_RUN_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort)
|
|
echo "Detailled information:"
|
|
echo "REF_ERROR = ${REF_ERROR}"
|
|
echo "CURRENT_RUN_ERROR = ${CURRENT_RUN_ERROR}"
|
|
echo "REF_FAILING = ${REF_FAILING}"
|
|
echo "CURRENT_RUN_FAILING = ${CURRENT_RUN_FAILING}"
|
|
|
|
# Compare failing and error tests
|
|
for LINE in ${CURRENT_RUN_FAILING}
|
|
do
|
|
if ! grep -Fxq ${LINE}<<<"${REF_FAILING}"
|
|
then
|
|
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
|
|
then
|
|
MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?"
|
|
echo "::error ::$MSG"
|
|
echo $MSG >> ${COMMENT_LOG}
|
|
have_new_failures="true"
|
|
else
|
|
MSG="Skip an intermittent issue ${LINE} (fails in this run but passes in the 'main' branch)"
|
|
echo "::warning ::$MSG"
|
|
echo $MSG >> ${COMMENT_LOG}
|
|
echo ""
|
|
fi
|
|
fi
|
|
done
|
|
|
|
for LINE in ${REF_FAILING}
|
|
do
|
|
if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_FAILING}"
|
|
then
|
|
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
|
|
then
|
|
MSG="Congrats! The gnu test ${LINE} is no longer failing!"
|
|
echo "::warning ::$MSG"
|
|
echo $MSG >> ${COMMENT_LOG}
|
|
else
|
|
MSG="Skipping an intermittent issue ${LINE} (passes in this run but fails in the 'main' branch)"
|
|
echo "::warning ::$MSG"
|
|
echo $MSG >> ${COMMENT_LOG}
|
|
echo ""
|
|
fi
|
|
fi
|
|
done
|
|
|
|
for LINE in ${CURRENT_RUN_ERROR}
|
|
do
|
|
if ! grep -Fxq ${LINE}<<<"${REF_ERROR}"
|
|
then
|
|
MSG="GNU test error: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?"
|
|
echo "::error ::$MSG"
|
|
echo $MSG >> ${COMMENT_LOG}
|
|
have_new_failures="true"
|
|
fi
|
|
done
|
|
|
|
for LINE in ${REF_ERROR}
|
|
do
|
|
if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_ERROR}"
|
|
then
|
|
MSG="Congrats! The gnu test ${LINE} is no longer ERROR!"
|
|
echo "::warning ::$MSG"
|
|
echo $MSG >> ${COMMENT_LOG}
|
|
fi
|
|
done
|
|
else
|
|
echo "::warning ::Skipping ${test_type} test failure comparison; no prior reference test logs are available."
|
|
fi
|
|
}
|
|
|
|
# Compare standard tests
|
|
compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' "${REF_LOG_FILE}" "standard"
|
|
|
|
# Compare root tests
|
|
compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite-root.log' "${ROOT_REF_LOG_FILE}" "root"
|
|
|
|
if test -n "${have_new_failures}" ; then exit -1 ; fi
|
|
- name: Upload comparison log (for GnuComment workflow)
|
|
if: success() || failure() # run regardless of prior step success/failure
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: comment
|
|
path: ${{ steps.vars.outputs.path_reference }}/comment/
|
|
- name: Compare test summary VS reference
|
|
if: success() || failure() # run regardless of prior step success/failure
|
|
shell: bash
|
|
run: |
|
|
## Compare test summary VS reference
|
|
REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json'
|
|
if test -f "${REF_SUMMARY_FILE}"; then
|
|
echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")"
|
|
mv "${REF_SUMMARY_FILE}" main-gnu-result.json
|
|
python uutils/util/compare_gnu_result.py
|
|
else
|
|
echo "::warning ::Skipping test summary comparison; no prior reference summary is available."
|
|
fi
|
|
|
|
gnu_coverage:
|
|
name: Run GNU tests with coverage
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
- name: Checkout code uutil
|
|
uses: actions/checkout@v4
|
|
with:
|
|
path: 'uutils'
|
|
- name: Checkout GNU coreutils
|
|
uses: actions/checkout@v4
|
|
with:
|
|
repository: 'coreutils/coreutils'
|
|
path: 'gnu'
|
|
ref: 'v9.5'
|
|
submodules: recursive
|
|
- uses: dtolnay/rust-toolchain@master
|
|
with:
|
|
toolchain: nightly
|
|
components: rustfmt
|
|
- uses: taiki-e/install-action@grcov
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
workspaces: "./uutils -> target"
|
|
- name: Install dependencies
|
|
run: |
|
|
## Install dependencies
|
|
sudo apt-get update
|
|
sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev
|
|
- name: Add various locales
|
|
run: |
|
|
## Add various locales
|
|
echo "Before:"
|
|
locale -a
|
|
## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory'
|
|
## Some others need a French locale
|
|
sudo locale-gen
|
|
sudo locale-gen --keep-existing fr_FR
|
|
sudo locale-gen --keep-existing fr_FR.UTF-8
|
|
sudo update-locale
|
|
echo "After:"
|
|
locale -a
|
|
- name: Build binaries
|
|
env:
|
|
RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
|
|
RUSTDOCFLAGS: "-Cpanic=abort"
|
|
run: |
|
|
## Build binaries
|
|
cd uutils
|
|
bash util/build-gnu.sh
|
|
- name: Run GNU tests
|
|
run: bash uutils/util/run-gnu-test.sh
|
|
- name: Generate coverage data (via `grcov`)
|
|
id: coverage
|
|
run: |
|
|
## Generate coverage data
|
|
cd uutils
|
|
COVERAGE_REPORT_DIR="target/debug"
|
|
COVERAGE_REPORT_FILE="${COVERAGE_REPORT_DIR}/lcov.info"
|
|
mkdir -p "${COVERAGE_REPORT_DIR}"
|
|
sudo chown -R "$(whoami)" "${COVERAGE_REPORT_DIR}"
|
|
# display coverage files
|
|
grcov . --output-type files --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" | sort --unique
|
|
# generate coverage report
|
|
grcov . --output-type lcov --output-path "${COVERAGE_REPORT_FILE}" --branch --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()"
|
|
echo "report=${COVERAGE_REPORT_FILE}" >> $GITHUB_OUTPUT
|
|
- name: Upload coverage results (to Codecov.io)
|
|
uses: codecov/codecov-action@v4
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
file: ${{ steps.coverage.outputs.report }}
|
|
flags: gnutests
|
|
name: gnutests
|
|
working-directory: uutils
|