From e5e8670bcb0e420a63c40cb936aed1089fe81c02 Mon Sep 17 00:00:00 2001 From: Johannes Altmanninger Date: Sun, 14 Apr 2024 20:53:26 +0200 Subject: [PATCH] Release automation script We provide releases in several different formats and channels. There are many manual steps involved in making a release. This makes it hard to learn or share improvements (especially share improvements that last once-and-for-all). Also there is potential for error -- for example our Debian PPAs don't include the latest release and our website doesn't link to the latest tarball. With our current approach, packaging tends to get more complicated, for example we'll want to add things like "Depends: librust-hexponent" to our Debian package, and set the version based on Cargo.toml. Add a script to automate the boring steps of the release process. This should obsolete most of https://github.com/fish-shell/fish-shell/wiki/Release-checklist It seems to work on my Linux system but it's obviously not battle-tested yet. The most essential TODO comments ones are for uploading Debian packages. Iteration on the script itself is usually quite fast. I run something like git add build_tools/release.sh git commit -m wip build_tools/release.sh 3.8.0 --dry-run --no-test # plus eventual --no-rebuild --no-rebuild-debs As a temporary hack, the Debian packages are currently built in a Docker container. It'd be easy to skip Docker when debmake is installed on the host system. We don't yet pass through gpg-agent into Docker, so the debuild process requires interactive password input. It takes a really long time (> 15 minutes) to build all Debian packages because we rebuild them sequentially for all Ubuntu releases (mantic, jammy, focal, bionic and xenial) even though only "debian/changelog" is different. For iterating on the script, we can use "--no-rebuild-debs" to reuse the the packages after they've been built once. Of course this is only valid if the tarball has not changed since. We could probably ask debuild to validate this, or speed up the process in a different way, at least parallelize. Future ideas: - we could run the script in CI instead, to keep us even more honest. - we should try to simplify the website release process (should not need to talk to the github API, the script already has enough information). - do something similar for the the macOS-specific parts --- .editorconfig | 4 + build_tools/release.sh | 273 ++++++++++++++++++ .../release/build-debian-package.Dockerfile | 47 +++ build_tools/release/build-debian-package.sh | 75 +++++ 4 files changed, 399 insertions(+) create mode 100755 build_tools/release.sh create mode 100644 build_tools/release/build-debian-package.Dockerfile create mode 100755 build_tools/release/build-debian-package.sh diff --git a/.editorconfig b/.editorconfig index 053c57353..2215e4356 100644 --- a/.editorconfig +++ b/.editorconfig @@ -18,6 +18,10 @@ trim_trailing_whitespace = false [*.{sh,ac}] indent_size = 2 +[{build_tools/release.sh,build_tools/release/*.sh}] +indent_size = 4 +max_line_length = 72 + [Dockerfile] indent_size = 2 diff --git a/build_tools/release.sh b/build_tools/release.sh new file mode 100755 index 000000000..4c2fba524 --- /dev/null +++ b/build_tools/release.sh @@ -0,0 +1,273 @@ +#!/bin/sh + +{ + +set -ex + +usage() { + cat << EOF +Usage: [FISH_GPG_KEY_ID=] $(basename $0) [OPTIONS] + +Build and publish a release with the given version. + +Release artifacts will be left behind in "./release" for debugging. + +On failure, an attempted is made to restore the state. +Same if --dry-run is given, even if there was no failure. + +$FISH_GPG_KEY_ID defaults to your Git author identity. + +Options: + -n, --dry-run Do not publish the release + --no-test Skip running tests. + --no-rebuild Do not rebuild tarball + --no-rebuild-debs Do not rebuild Debian packages +EOF + exit 1 +} + +build_debian_packages() { + dockerfile=$PWD/build_tools/release/build-debian-package.Dockerfile + + export DOCKER_BUILDKIT=1 + + # Construct a docker image. + img_tagname="ghcr.io/fish-shell/fish-ci/$(basename -s .Dockerfile "$dockerfile"):latest" + sudo=sudo + if groups | grep -q '\bdocker\b'; then + sudo= + fi + $sudo docker build \ + -t "$img_tagname" \ + -f "$dockerfile" \ + "$PWD"/build_tools/release/ + + tmpdir=$(mktemp -d) + $sudo docker run -it \ + --cidfile="$tmpdir/container_id" \ + --mount type=bind,source="$PWD",target=/fish-source,readonly \ + --mount type=bind,source="${GNUPGHOME:-"$HOME/.gnupg"}",target=/gnupg,readonly \ + --env "FISH_COMMITTER=$committer" \ + --env "FISH_GPG_KEY_ID=$gpg_key_id" \ + --env "FISH_VERSION=$version" \ + "$img_tagname" + + container_id=$(cat "$tmpdir/container_id") + + $sudo docker cp "$container_id":/home/fishuser/dpkgs release/dpkgs + $sudo chown -R $USER release/dpkgs + $sudo docker rm "$container_id" + $sudo rm -r "$tmpdir" +} + +committer="$(git var GIT_AUTHOR_IDENT)" +committer=${committer% *} # strip timezone +committer=${committer% *} # strip timestamp +gpg_key_id=${FISH_GPG_KEY_ID:-"$committer"} + +# TODO Check for user errors here (in case HEAD is not pushed yet). +is_latest_release=false +if git merge-base --is-ancestor origin/master HEAD; then + is_latest_release=true +fi + +version= +dry_run= +do_test=true +do_rebuild=true +do_rebuild_debs=true +for arg +do + case "$arg" in + (-n|--dry-run) dry_run=: ;; + (--no-test) do_test=false ;; + (--no-rebuild) do_rebuild=false ;; + (--no-rebuild-debs) do_rebuild_debs=false ;; + (-*) usage ;; + (*) + if [ -n "$version" ]; then + usage + fi + version=$arg + ;; + esac +done + +if [ -z "$version" ]; then + usage +fi + +for tool in \ + gh \ + gpg \ + osc \ + pip \ + sphinx-build \ + virtualenv \ +; do + if ! command -v "$tool" >/dev/null; then + echo >&2 "$0: missing command: $1" + exit 1 + fi +done + +if ! echo "$version" | grep '^3\.'; then + echo >&2 "$0: major version bump needs changes to PPA code" + exit 1 +fi + +if [ -e release ] && $do_rebuild ; then + echo >&2 "$0: ./release exists, please remove it or pass --no-rebuild to reuse the tarball" + exit 1 +fi + +for repo in . ../fish-site +do + if ! git -C "$repo" diff HEAD --quiet; then + echo >&2 "$0: index and worktree must be clean" + exit 1 + fi +done + +if git tag | grep -qxF "$version"; then + echo >&2 "$0: tag $version already exists" + exit 1 +fi + +cleanup_done=false +did_commit=false +cleanup_after_error_or_dry_run() { + printf '\n\n' + if $cleanup_done; then + return + fi + cleanup_done=true + { + set +e + if $did_commit; then + # Try to only undo our changelog changes. + git revert --no-edit HEAD && + git reset HEAD~2 + fi + git tag -d $version + exit 1 + } >/dev/null +} +trap cleanup_after_error_or_dry_run INT EXIT + +if $do_test; then + ninja -Cbuild test +fi + +sed -i "1cfish $version (released $(date +'%B %d, %Y'))" CHANGELOG.rst +git add CHANGELOG.rst +git commit -m "Release $version" +did_commit=true # TODO This is hacky. +git -c "user.signingKey=$gpg_key_id" tag -a --message="Release $version" $version -s + +if ! [ -d release ] || $do_rebuild; then + mkdir -p release + FISH_ARTEFACT_PATH=$PWD/release build_tools/make_tarball.sh + gpg --local-user "$gpg_key_id" --sign --detach --armor "release/fish-$version.tar.xz" +fi + +if ! [ -d release/dpkgs ] || $do_rebuild_debs; then + build_debian_packages +fi + +rm -rf release/obs +mkdir -p release/obs +( + cd release/obs + osc checkout shells:fish:release:3/fish + cd shells:fish:release:3/fish + rm -f *.tar.* *.dsc +) +ln release/dpkgs/fish_$version.orig.tar.xz release/obs/shells:fish:release:3/fish +ln release/dpkgs/fish_$version-1.debian.tar.xz release/obs/shells:fish:release:3/fish +ln release/dpkgs/fish_$version-1.dsc release/obs/shells:fish:release:3/fish +rpmversion=$(echo $version | sed -e 's/-/+/' -e 's/-/./g') +sed -e "s/@version@/$version/g" -e "s/@RPMVERSION@/$rpmversion/g" \ + < "release/dpkgs/fish-$version/fish.spec.in" > release/obs/shells:fish:release:3/fish/fish.spec + +( + cd release/obs/shells:fish:release:3/fish + osc addremove +) + +relnotes_tmp=$(mktemp -d) +( + virtualenv release/.venv + . release/.venv/bin/activate + pip install sphinx-markdown-builder + # We only need to build latest relnotes. I'm not sure if that's + # possible, so resort to building everything. + mkdir $relnotes_tmp/src $relnotes_tmp/out + sphinx-build -j 8 -b markdown -n "release/dpkgs/fish-$version/doc_src" \ + -d "release/dpkgs/fish-$version/user_doc/doctrees" \ + $relnotes_tmp/markdown + # Delete title + sed -i 1,3d "$relnotes_tmp/markdown/relnotes.md" + # Delete notes for prior releases. + sed -i '/^## /,$d' "$relnotes_tmp/markdown/relnotes.md" +) + +if ! [ -n $dry_run ]; then + trap '' INT EXIT +fi +$dry_run git push origin "$version" # Push the tag +( + cd release/obs + $dry_run osc commit -m "New release: $version" +) +$dry_run gh release create --draft \ + --title=$(head -n 1 "release/dpkgs/fish-$version/CHANGELOG.rst") \ + --notes-file="$relnotes_tmp/markdown/relnotes.md" \ + --verify-tag \ + "$version" \ + "release/fish-$version.tar.xz" "release/fish-$version.tar.xz.asc" +if $is_latest_release; then + $dry_run git push origin $version:master +fi +rm -rf $relnotes_tmp + +(set +x +echo "\ +To update fish-site we currently query the github release API +Please publish the draft at https://github.com/fish-shell/fish-shell/releases/tag/$version +and hit Enter to continue with updating ../fish-site") +read line + +minor_version=${version%.*} +if $is_latest_release; then + rm -rf "../fish-site/site/docs/current" + cp -r "release/dpkgs/fish-$version/user_doc/html" ../fish-site/site/docs/current +fi +rm -rf "../fish-site/site/docs/$minor_version" +cp -r "release/dpkgs/fish-$version/user_doc/html" "../fish-site/site/docs/$minor_version" +( + cd ../fish-site + make new-release + git add -u + git add site/docs/$minor_version docs/docs/$minor_version + git add site/docs/current docs/docs/current + git commit --message="Release $version" + $dry_run git push + if [ -n $dry_run ]; then + git reset --hard HEAD~ >/dev/null + fi +) + +cat < + ./build_tools/mac_notarize.sh ~/fish_built/fish-$version.app.zip +- Add macOS packages to https://github.com/fish-shell/fish-shell/releases/tag/$version +- Send email to "fish-users Mailing List " +EOF + +exit + +} diff --git a/build_tools/release/build-debian-package.Dockerfile b/build_tools/release/build-debian-package.Dockerfile new file mode 100644 index 000000000..19d09f200 --- /dev/null +++ b/build_tools/release/build-debian-package.Dockerfile @@ -0,0 +1,47 @@ +# Currently need testing for rustc>=1.67 +FROM debian:testing +LABEL org.opencontainers.image.source=https://github.com/fish-shell/fish-shell + +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get -y install \ + build-essential \ + cargo \ + cmake \ + debhelper \ + debmake \ + file \ + g++ \ + gettext \ + git \ + libpcre2-dev \ + locales \ + locales-all \ + ninja-build \ + pkg-config \ + python3 \ + python3-pexpect \ + python3-launchpadlib \ + rustc \ + sudo \ + tmux \ + && locale-gen en_US.UTF-8 \ + && apt-get clean + +RUN groupadd -g 1000 fishuser \ + && useradd -p $(openssl passwd -1 fish) -d /home/fishuser -m -u 1000 -g 1000 fishuser \ + && adduser fishuser sudo \ + && mkdir /fish-source \ + && chown -R fishuser:fishuser /home/fishuser /fish-source + +USER fishuser +WORKDIR /home/fishuser + +COPY /build-debian-package.sh / + +CMD cp -r /gnupg ~/.gnupg && \ + /build-debian-package.sh /fish-source /fish-source/release /home/fishuser/dpkgs || \ + bash diff --git a/build_tools/release/build-debian-package.sh b/build_tools/release/build-debian-package.sh new file mode 100755 index 000000000..ae0bb3b17 --- /dev/null +++ b/build_tools/release/build-debian-package.sh @@ -0,0 +1,75 @@ +#!/bin/sh + +set -eux + +source=$1 +tarball_dir=$2 +output=$3 + +committer=$FISH_COMMITTER # TODO Make sure that this person is listed in debian/control ? +gpg_key_id=$FISH_GPG_KEY_ID +version=$FISH_VERSION + +export DEBEMAIL=$committer +export EDITOR=true + +mkdir "$output" +# Should be a hard link. +cp "$tarball_dir/fish-$version.tar.xz" "$output/fish_$version.orig.tar.xz" +( + cd "$output" + tar xf "fish_$version.orig.tar.xz" +) +tarball_sha256sum=$(sha256sum "$output/fish-$version.tar.xz" | cut -d' ' -f1) + +# Copy in the Debian packaging information +cp -r "$source/debian" "$output"/fish-$version/debian + +# TODO Include old changes? +echo > "$output"/fish-$version/debian/changelog "\ +fish ($version-1) testing; urgency=medium + + * Upstream release. + + -- $committer $(date)" + +# Update the Debian changelog for the new version - interactive +( + cd "$output"/fish-$version + dch --newversion "$version-1" --distribution testing +) + +( + cd "$output" + ( + cd fish-$version + # Build the package for Debian on OBS. + debuild -S -k"$gpg_key_id" + # Set up an array for the Ubuntu packages + ppa_series=$( + echo ' + from launchpadlib.launchpad import Launchpad + launchpad = Launchpad.login_anonymously("fish shell build script", "production", "~/.cache", version="devel") + ubu = launchpad.projects("ubuntu") + # dash requires a double backslash for unknown reasons. + print("\\n".join(x["name"] for x in ubu.series.entries if x["supported"] == True and x["name"] not in ("trusty", "precise"))) + ' | + sed s,^[[:space:]]*,, | + python3 + ) + # Build the Ubuntu source packages + for series in $ppa_series + do + sed -i -e "s/^fish ($version-1)/fish ($version-1~$series)/" -e "s/testing/$series/" debian/changelog + debuild -S -sa -k"$gpg_key_id" + sed -i -e "s/^fish ($version-1~$series)/fish ($version-1)/" -e "s/$series/testing/" debian/changelog + done + ) + + # TODO populate the dput config file. + # # Or other appropriate PPA defined in ~/.dput.cf + # for i in fish_$version-1~*.changes + # do + # dput fish-release-3 "$i" + # done +)