From 8d19c65eb1460379f10d3655719e3bec1aceeb00 Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Fri, 11 Oct 2019 18:11:22 -0700 Subject: [PATCH 01/14] Use codeclimate for test coverage, v1 Adds a coverage.sh to be executed by the coverage pipeline. Signed-off-by: Miah Johnson --- .expeditor/buildkite/coverage.sh | 60 ++++++++++++++++++++++++++++++++ .expeditor/coverage.pipeline.yml | 3 +- 2 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 .expeditor/buildkite/coverage.sh diff --git a/.expeditor/buildkite/coverage.sh b/.expeditor/buildkite/coverage.sh new file mode 100644 index 000000000..5f5368915 --- /dev/null +++ b/.expeditor/buildkite/coverage.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +set -ueo pipefail + +export LANG=C.UTF-8 LANGUAGE=C.UTF-8 +# test-reporter expects reporter identifier under this environment variable +CC_TEST_REPORTER_ID="FIXME" +TEST_REPORTER_VERSION="0.6.3" +S3_URL="s3://public-cd-buildkite-cache/${BUILDKITE_PIPELINE_SLUG}/${BUILDKITE_LABEL}" + +download_test_reporter() { + curl -o test-reporter -L https://codeclimate.com/downloads/test-reporter/test-reporter-"${TEST_REPORTER_VERSION}"-linux-amd64 + chmod +x test-reporter +} + +download_s3_file() { + aws s3 cp "${S3_URL}/$1" "$1" +} + +upload_s3_file() { + if [ -f "$1" ]; then + aws s3 cp "$1" "${S3_URL}/$1" || echo "Could not push $1 to S3 for caching." + fi +} + +echo "--- downloading coverage tool" +aws s3 cp "${S3_URL}/test-reporter" || download_test_reporter +aws s3 cp "${S3_URL}/test-reporter.sha" || echo "Could not download test-reporter.sha" + + +echo "--- updating rubygems" +gem update --system -N + +echo "--- system details" +uname -a +gem env +bundle --version + +echo "--- setting up test coverage before build" +./test-reporter before-build + +echo "--- bundle install" +bundle install --jobs=7 --retry=3 --without tools maintenance deploy + +echo "+++ bundle exec rake" +bundle exec rake "${RAKE_TASK:-}" +EXIT-CODE=$? + +echo "+++ formatting and uploading test coverage" +./test-reporter after-build -t simplecov --exit-code $EXIT_CODE + +echo "--- uploading test-reporter{.sha} to s3" +if [ "test-reporter" -nt "test-reporter.sha" ]; then + if shasum --check test-reporter.sha --status; then + shasum -a 256 test-reporter > test-reporter.sha + for i in "test-reporter" "test-reporter.sha"; do + upload_s3_file "$i" + done + fi +fi diff --git a/.expeditor/coverage.pipeline.yml b/.expeditor/coverage.pipeline.yml index ba44dddbe..1cda067f5 100644 --- a/.expeditor/coverage.pipeline.yml +++ b/.expeditor/coverage.pipeline.yml @@ -9,8 +9,7 @@ steps: - label: coverage commands: - - bundle install --jobs=7 --retry=3 --without tools maintenance deploy - - bundle exec rake test:default + - /workdir/.expeditor/buildkite/coverage.sh expeditor: executor: docker: From 2a00b2b6a7a3942a9bcffe29d91baf44663823c7 Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Thu, 17 Oct 2019 16:37:47 -0700 Subject: [PATCH 02/14] Clean up variable usage, and use download_s3_file Signed-off-by: Miah Johnson --- .expeditor/buildkite/coverage.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.expeditor/buildkite/coverage.sh b/.expeditor/buildkite/coverage.sh index 5f5368915..5d33b1a34 100644 --- a/.expeditor/buildkite/coverage.sh +++ b/.expeditor/buildkite/coverage.sh @@ -6,26 +6,26 @@ export LANG=C.UTF-8 LANGUAGE=C.UTF-8 # test-reporter expects reporter identifier under this environment variable CC_TEST_REPORTER_ID="FIXME" TEST_REPORTER_VERSION="0.6.3" -S3_URL="s3://public-cd-buildkite-cache/${BUILDKITE_PIPELINE_SLUG}/${BUILDKITE_LABEL}" +S3_URL="s3://public-cd-buildkite-cache/$BUILDKITE_PIPELINE_SLUG/$BUILDKITE_LABEL" download_test_reporter() { - curl -o test-reporter -L https://codeclimate.com/downloads/test-reporter/test-reporter-"${TEST_REPORTER_VERSION}"-linux-amd64 + curl -o test-reporter -L https://codeclimate.com/downloads/test-reporter/test-reporter-"$TEST_REPORTER_VERSION"-linux-amd64 chmod +x test-reporter } download_s3_file() { - aws s3 cp "${S3_URL}/$1" "$1" + aws s3 cp "$S3_URL/$1" "$1" } upload_s3_file() { if [ -f "$1" ]; then - aws s3 cp "$1" "${S3_URL}/$1" || echo "Could not push $1 to S3 for caching." + aws s3 cp "$1" "$S3_URL/$1" || echo "Could not push $1 to S3 for caching." fi } echo "--- downloading coverage tool" -aws s3 cp "${S3_URL}/test-reporter" || download_test_reporter -aws s3 cp "${S3_URL}/test-reporter.sha" || echo "Could not download test-reporter.sha" +download_s3_file test-reporter || download_test_reporter +download_s3_file test-reporter.sha || echo "Could not download test-reporter.sha" echo "--- updating rubygems" @@ -47,9 +47,9 @@ bundle exec rake "${RAKE_TASK:-}" EXIT-CODE=$? echo "+++ formatting and uploading test coverage" -./test-reporter after-build -t simplecov --exit-code $EXIT_CODE +./test-reporter after-build -t simplecov --exit-code "$EXIT_CODE" -echo "--- uploading test-reporter{.sha} to s3" +echo "--- uploading test-reporter.sha to s3" if [ "test-reporter" -nt "test-reporter.sha" ]; then if shasum --check test-reporter.sha --status; then shasum -a 256 test-reporter > test-reporter.sha From 38a027350a370e129b54b09a97b81a6cd798d69a Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Wed, 27 Nov 2019 20:26:27 -0800 Subject: [PATCH 03/14] Set up our coverage secrets Signed-off-by: Miah Johnson --- .expeditor/buildkite/coverage.sh | 3 ++- .expeditor/coverage.pipeline.yml | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.expeditor/buildkite/coverage.sh b/.expeditor/buildkite/coverage.sh index 5d33b1a34..56ec95310 100644 --- a/.expeditor/buildkite/coverage.sh +++ b/.expeditor/buildkite/coverage.sh @@ -4,7 +4,8 @@ set -ueo pipefail export LANG=C.UTF-8 LANGUAGE=C.UTF-8 # test-reporter expects reporter identifier under this environment variable -CC_TEST_REPORTER_ID="FIXME" +CC_TEST_REPORTER_ID="$COVERAGE_ID" +export CC_TEST_REPORTER_ID TEST_REPORTER_VERSION="0.6.3" S3_URL="s3://public-cd-buildkite-cache/$BUILDKITE_PIPELINE_SLUG/$BUILDKITE_LABEL" diff --git a/.expeditor/coverage.pipeline.yml b/.expeditor/coverage.pipeline.yml index 1cda067f5..659d9943b 100644 --- a/.expeditor/coverage.pipeline.yml +++ b/.expeditor/coverage.pipeline.yml @@ -11,6 +11,10 @@ steps: commands: - /workdir/.expeditor/buildkite/coverage.sh expeditor: + secrets: + COVERAGE_ID: + path: secret/coveralls/inspec/inspec + field: reporter_id executor: docker: image: ruby:2.6-stretch From 4878df9e0d1ffb71e66bd9483625cf980b9faf7e Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Wed, 27 Nov 2019 20:30:23 -0800 Subject: [PATCH 04/14] let releng own the docker image Signed-off-by: Miah Johnson --- .expeditor/coverage.pipeline.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.expeditor/coverage.pipeline.yml b/.expeditor/coverage.pipeline.yml index 659d9943b..8c6b8fd43 100644 --- a/.expeditor/coverage.pipeline.yml +++ b/.expeditor/coverage.pipeline.yml @@ -15,6 +15,3 @@ steps: COVERAGE_ID: path: secret/coveralls/inspec/inspec field: reporter_id - executor: - docker: - image: ruby:2.6-stretch From 67a4ff060aa6725c76642e649c9773b6c0d5e214 Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Wed, 27 Nov 2019 20:36:32 -0800 Subject: [PATCH 05/14] update path Signed-off-by: Miah Johnson --- .expeditor/coverage.pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.expeditor/coverage.pipeline.yml b/.expeditor/coverage.pipeline.yml index 8c6b8fd43..c63e71831 100644 --- a/.expeditor/coverage.pipeline.yml +++ b/.expeditor/coverage.pipeline.yml @@ -9,7 +9,7 @@ steps: - label: coverage commands: - - /workdir/.expeditor/buildkite/coverage.sh + - .expeditor/buildkite/coverage.sh expeditor: secrets: COVERAGE_ID: From b985ddc04140b237dcbefb3a37dbc04a5f2b97c6 Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Wed, 27 Nov 2019 20:43:03 -0800 Subject: [PATCH 06/14] this uses chefes/buildkite Signed-off-by: Miah Johnson --- .expeditor/coverage.pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.expeditor/coverage.pipeline.yml b/.expeditor/coverage.pipeline.yml index c63e71831..45630e379 100644 --- a/.expeditor/coverage.pipeline.yml +++ b/.expeditor/coverage.pipeline.yml @@ -11,6 +11,8 @@ steps: commands: - .expeditor/buildkite/coverage.sh expeditor: + executor: + docker: secrets: COVERAGE_ID: path: secret/coveralls/inspec/inspec From 726be2fa7ac5cec39bac7193c0a084845a0700bc Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Wed, 27 Nov 2019 20:45:57 -0800 Subject: [PATCH 07/14] update perm Signed-off-by: Miah Johnson --- .expeditor/buildkite/coverage.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .expeditor/buildkite/coverage.sh diff --git a/.expeditor/buildkite/coverage.sh b/.expeditor/buildkite/coverage.sh old mode 100644 new mode 100755 From 144081eab5b316a716bf4b751226743afd8a73ba Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Wed, 27 Nov 2019 20:51:53 -0800 Subject: [PATCH 08/14] execute test Signed-off-by: Miah Johnson --- .expeditor/buildkite/coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.expeditor/buildkite/coverage.sh b/.expeditor/buildkite/coverage.sh index 56ec95310..7d73cff13 100755 --- a/.expeditor/buildkite/coverage.sh +++ b/.expeditor/buildkite/coverage.sh @@ -44,7 +44,7 @@ echo "--- bundle install" bundle install --jobs=7 --retry=3 --without tools maintenance deploy echo "+++ bundle exec rake" -bundle exec rake "${RAKE_TASK:-}" +bundle exec rake test EXIT-CODE=$? echo "+++ formatting and uploading test coverage" From d3b7a82cd6b436843370d79482e54b0429f7cca8 Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Wed, 27 Nov 2019 21:04:44 -0800 Subject: [PATCH 09/14] trigger coverage on pull requests Signed-off-by: Miah Johnson --- .expeditor/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.expeditor/config.yml b/.expeditor/config.yml index c5d504251..97288d1ba 100644 --- a/.expeditor/config.yml +++ b/.expeditor/config.yml @@ -162,6 +162,7 @@ subscriptions: - built_in:notify_chefio_slack_channels - workload: pull_request_opened:{{agent_id}}:* actions: + - trigger_pipeline:coverage - post_github_comment:.expeditor/templates/pull_request.mustache: ignore_team_members: - inspec/owners From 9db7b70e30c16b0be212917ec1094b6b9272a482 Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Wed, 27 Nov 2019 21:13:47 -0800 Subject: [PATCH 10/14] remove coveralls; all we need is simplecov Signed-off-by: Miah Johnson --- .expeditor/buildkite/coverage.sh | 2 +- Gemfile | 1 - test/helper.rb | 6 ++---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.expeditor/buildkite/coverage.sh b/.expeditor/buildkite/coverage.sh index 7d73cff13..b36521e7e 100755 --- a/.expeditor/buildkite/coverage.sh +++ b/.expeditor/buildkite/coverage.sh @@ -45,7 +45,7 @@ bundle install --jobs=7 --retry=3 --without tools maintenance deploy echo "+++ bundle exec rake" bundle exec rake test -EXIT-CODE=$? +EXIT_CODE=$? echo "+++ formatting and uploading test coverage" ./test-reporter after-build -t simplecov --exit-code "$EXIT_CODE" diff --git a/Gemfile b/Gemfile index 7cfd83b3d..d46320efc 100644 --- a/Gemfile +++ b/Gemfile @@ -20,7 +20,6 @@ end group :test do gem "chefstyle", "~> 0.13.0" - gem "coveralls", require: false gem "minitest", "~> 5.5" gem "minitest-sprint", "~> 1.0" gem "rake", ">= 10" diff --git a/test/helper.rb b/test/helper.rb index 6849ea5ad..fd9975104 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -2,17 +2,15 @@ # Do not add any code above this line. ## -# Do not add any other code to this code block. Simplecov and -# coveralls only until the next code block: +# Do not add any other code to this code block. Simplecov +# only until the next code block: if ENV["CI_ENABLE_COVERAGE"] require "simplecov/no_defaults" require "helpers/simplecov_minitest" - require "coveralls" SimpleCov.formatters = SimpleCov::Formatter::MultiFormatter.new([ SimpleCov::Formatter::HTMLFormatter, - Coveralls::SimpleCov::Formatter, ]) SimpleCov.start do From 12329cf48a555d001eeb03c2263cb22ac72a82a2 Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Wed, 27 Nov 2019 22:38:20 -0800 Subject: [PATCH 11/14] this was listed twice? Signed-off-by: Miah Johnson --- .expeditor/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.expeditor/config.yml b/.expeditor/config.yml index 97288d1ba..583a3433d 100644 --- a/.expeditor/config.yml +++ b/.expeditor/config.yml @@ -171,4 +171,3 @@ subscriptions: only_if_team_member: - inspec/owners - inspec/inspec-core-team - - post_github_comment:.expeditor/templates/pull_request.mustache From 6d0abf298094a0638c24eaf87b8806a98512617e Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Wed, 27 Nov 2019 22:38:32 -0800 Subject: [PATCH 12/14] clean up output Signed-off-by: Miah Johnson --- .expeditor/buildkite/coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.expeditor/buildkite/coverage.sh b/.expeditor/buildkite/coverage.sh index b36521e7e..aa5b22457 100755 --- a/.expeditor/buildkite/coverage.sh +++ b/.expeditor/buildkite/coverage.sh @@ -26,7 +26,7 @@ upload_s3_file() { echo "--- downloading coverage tool" download_s3_file test-reporter || download_test_reporter -download_s3_file test-reporter.sha || echo "Could not download test-reporter.sha" +download_s3_file test-reporter.sha || echo -e "\nCould not download test-reporter.sha" echo "--- updating rubygems" From 64e444883c2fc50f841079665b01b86226591a89 Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Wed, 27 Nov 2019 22:43:27 -0800 Subject: [PATCH 13/14] change order? Signed-off-by: Miah Johnson --- .expeditor/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.expeditor/config.yml b/.expeditor/config.yml index 583a3433d..5aed254e2 100644 --- a/.expeditor/config.yml +++ b/.expeditor/config.yml @@ -162,7 +162,6 @@ subscriptions: - built_in:notify_chefio_slack_channels - workload: pull_request_opened:{{agent_id}}:* actions: - - trigger_pipeline:coverage - post_github_comment:.expeditor/templates/pull_request.mustache: ignore_team_members: - inspec/owners @@ -171,3 +170,4 @@ subscriptions: only_if_team_member: - inspec/owners - inspec/inspec-core-team + - trigger_pipeline:coverage From fe8f86850485ca333ed1d6864f53f448dda6b653 Mon Sep 17 00:00:00 2001 From: Miah Johnson Date: Thu, 28 Nov 2019 00:53:22 -0800 Subject: [PATCH 14/14] dont set a default simplecov format; we do that elsewhere. Signed-off-by: Miah Johnson --- .expeditor/buildkite/coverage.sh | 22 +++++++++++++++------- test/helper.rb | 4 ---- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.expeditor/buildkite/coverage.sh b/.expeditor/buildkite/coverage.sh index aa5b22457..4418d308a 100755 --- a/.expeditor/buildkite/coverage.sh +++ b/.expeditor/buildkite/coverage.sh @@ -12,6 +12,7 @@ S3_URL="s3://public-cd-buildkite-cache/$BUILDKITE_PIPELINE_SLUG/$BUILDKITE_LABEL download_test_reporter() { curl -o test-reporter -L https://codeclimate.com/downloads/test-reporter/test-reporter-"$TEST_REPORTER_VERSION"-linux-amd64 chmod +x test-reporter + touch new_test-reporter } download_s3_file() { @@ -51,11 +52,18 @@ echo "+++ formatting and uploading test coverage" ./test-reporter after-build -t simplecov --exit-code "$EXIT_CODE" echo "--- uploading test-reporter.sha to s3" -if [ "test-reporter" -nt "test-reporter.sha" ]; then - if shasum --check test-reporter.sha --status; then - shasum -a 256 test-reporter > test-reporter.sha - for i in "test-reporter" "test-reporter.sha"; do - upload_s3_file "$i" - done - fi +if [ -f "new_test-reporter" ]; then + echo "new test-reporter detected. uploading." + shasum -a 256 test-reporter > test-reporter.sha + for i in "test-reporter" "test-reporter.sha"; do + upload_s3_file "$i" + done +fi + +if shasum --check test-reporter.sha --status; then + echo "test-reporter shasum mismatch. uploading." + shasum -a 256 test-reporter > test-reporter.sha + for i in "test-reporter" "test-reporter.sha"; do + upload_s3_file "$i" + done fi diff --git a/test/helper.rb b/test/helper.rb index fd9975104..9f4366d86 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -9,10 +9,6 @@ if ENV["CI_ENABLE_COVERAGE"] require "simplecov/no_defaults" require "helpers/simplecov_minitest" - SimpleCov.formatters = SimpleCov::Formatter::MultiFormatter.new([ - SimpleCov::Formatter::HTMLFormatter, - ]) - SimpleCov.start do add_filter "/test/" add_group "Resources", ["lib/resources", "lib/inspec/resources"]