From e5318f35640eaae3b98b7dcc9212ddc5d4c8a240 Mon Sep 17 00:00:00 2001 From: Ryan Davis Date: Wed, 4 Dec 2019 15:34:38 -0800 Subject: [PATCH] Refactor BK verify script so we can reuse bundle caching across projects easily. Signed-off-by: Ryan Davis --- .expeditor/buildkite/cache_support.sh | 58 +++++++++++++++++++++++++++ .expeditor/buildkite/verify.sh | 57 +++++--------------------- 2 files changed, 67 insertions(+), 48 deletions(-) create mode 100755 .expeditor/buildkite/cache_support.sh diff --git a/.expeditor/buildkite/cache_support.sh b/.expeditor/buildkite/cache_support.sh new file mode 100755 index 000000000..61fdfe5f5 --- /dev/null +++ b/.expeditor/buildkite/cache_support.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +set -ue + +S3_URL="s3://public-cd-buildkite-cache/${BUILDKITE_PIPELINE_SLUG}/${BUILDKITE_LABEL}" + +pull_s3_file() { + aws s3 cp "${S3_URL}/$1" "$1" || echo "Could not pull $1 from S3" +} + +push_s3_file() { + if [ -f "$1" ]; then + aws s3 cp "$1" "${S3_URL}/$1" || echo "Could not push $1 to S3 for caching." + fi +} + +install_cache_deps() { + apt-get update -y + + if [ -n "${1:-}" ]; then + apt-get install "$@" -y + fi + + if [ -z "${SKIP_BUNDLE_CACHE:-}" ]; then + apt-get install awscli -y + fi +} + +pull_bundle() { + if [ -z "${SKIP_BUNDLE_CACHE:-}" ]; then + pull_s3_file "bundle.tar.gz" + pull_s3_file "bundle.sha256" + + if [ -f bundle.tar.gz ]; then + tar -xzf bundle.tar.gz + mv Gemfile.lock Gemfile.lock.old || true + fi + + if [ -n "${RESET_BUNDLE_CACHE:-}" ]; then + rm bundle.sha256 + fi + fi +} + +push_bundle() { + if [ -z "${SKIP_BUNDLE_CACHE:-}" ]; then + if test -f bundle.sha256 && shasum --check bundle.sha256 --status; then + echo "Bundled gems have not changed. Skipping upload to s3" + else + echo "Bundled gems have changed. Uploading to s3" + diff -u Gemfile.lock.old Gemfile.lock || true + shasum -a 256 Gemfile.lock > bundle.sha256 + tar -czf bundle.tar.gz Gemfile.lock vendor/ + push_s3_file bundle.tar.gz + push_s3_file bundle.sha256 + fi + fi +} diff --git a/.expeditor/buildkite/verify.sh b/.expeditor/buildkite/verify.sh index df2138423..2831882c5 100755 --- a/.expeditor/buildkite/verify.sh +++ b/.expeditor/buildkite/verify.sh @@ -1,29 +1,11 @@ #!/bin/bash -set -ue - echo "--- dependencies" -export LANG=C.UTF-8 LANGUAGE=C.UTF-8 -S3_URL="s3://public-cd-buildkite-cache/${BUILDKITE_PIPELINE_SLUG}/${BUILDKITE_LABEL}" - -pull_s3_file() { - aws s3 cp "${S3_URL}/$1" "$1" || echo "Could not pull $1 from S3" -} - -push_s3_file() { - if [ -f "$1" ]; then - aws s3 cp "$1" "${S3_URL}/$1" || echo "Could not push $1 to S3 for caching." - fi -} - -apt-get update -y -apt-get install sudo -y - -if [ -z "${SKIP_BUNDLE_CACHE:-}" ]; then - apt-get install awscli -y -fi +. .expeditor/buildkite/cache_support.sh +install_cache_deps sudo echo "--- setting up user" +export LANG=C.UTF-8 LANGUAGE=C.UTF-8 useradd -m -U --uid 2000 normal echo "normal ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/normal @@ -35,40 +17,19 @@ uname -a gem env bundle --version -echo "--- bundle install" -if [ -z "${SKIP_BUNDLE_CACHE:-}" ]; then - pull_s3_file "bundle.tar.gz" - pull_s3_file "bundle.sha256" - - if [ -f bundle.tar.gz ]; then - tar -xzf bundle.tar.gz - mv Gemfile.lock Gemfile.lock.old || true - fi - - if [ -n "${RESET_BUNDLE_CACHE:-}" ]; then - rm bundle.sha256 - fi -fi +echo "--- pull bundle cache" +pull_bundle +echo "--- bundle" bundle config --local path vendor/bundle bundle install --jobs=7 --retry=3 --without tools maintenance deploy -echo "--- bundle cache" -if [ -z "${SKIP_BUNDLE_CACHE:-}" ]; then - if test -f bundle.sha256 && shasum --check bundle.sha256 --status; then - echo "Bundled gems have not changed. Skipping upload to s3" - else - echo "Bundled gems have changed. Uploading to s3" - diff -u Gemfile.lock.old Gemfile.lock || true - shasum -a 256 Gemfile.lock > bundle.sha256 - tar -czf bundle.tar.gz Gemfile.lock vendor/ - push_s3_file bundle.tar.gz - push_s3_file bundle.sha256 - fi -fi +echo "--- push bundle cache" +push_bundle echo "+++ bundle exec rake ${RAKE_TASK:-}" # TODO: run this as non-root: # chown -R normal:normal /home/normal /workdir # su normal -c "bundle exec rake ${RAKE_TASK:-}" +# shellcheck disable=SC2086 bundle exec rake ${RAKE_TASK:-}