ci: automatically create and get build artifacts from Cirrus CI (#854)

This automatically triggers and grabs the build artifacts for systems that are only supported on Cirrus CI (as of now, FreeBSD and M1 macOS).

* ci: add cirrus build trigger script

* ci: modify build scripts to include cirrus build

* fix some stuff

* update docs

* more fixes
This commit is contained in:
Clement Tsang 2022-10-27 06:27:04 -04:00 committed by GitHub
parent 5eba26f9e5
commit 51498e1238
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 226 additions and 4 deletions

View file

@ -1,3 +1,5 @@
%YAML 1.1
---
# Configuration for CirrusCI. This is primarily used for
# FreeBSD and macOS M1 tests and builds.
@ -35,7 +37,8 @@ env:
CARGO_HUSKY_DONT_INSTALL_HOOKS: true
test_task:
only_if: $CIRRUS_CRON == "" && ($CIRRUS_BRANCH == "master" || $CIRRUS_PR != "")
auto_cancellation: false
only_if: $CIRRUS_BUILD_SOURCE != "api" && ($CIRRUS_BRANCH == "master" || $CIRRUS_PR != "")
matrix:
- name: "FreeBSD 13 Test"
freebsd_instance:
@ -60,7 +63,8 @@ test_task:
<<: *CLEANUP_TEMPLATE
build_task:
only_if: $CIRRUS_RELEASE != "" || $CIRRUS_CRON == "nightly" || $CIRRUS_API_CREATED == true || $CIRRUS_BRANCH == "master"
auto_cancellation: false
only_if: $CIRRUS_BUILD_SOURCE == "api"
env:
BTM_GENERATE: true
COMPLETION_DIR: "target/tmp/bottom/completion/"

View file

@ -8,6 +8,12 @@ name: "Build Releases"
on:
workflow_dispatch:
workflow_call:
inputs:
caller:
description: "The calling workflow."
default: ""
required: false
type: string
env:
CARGO_INCREMENTAL: 0
@ -249,6 +255,32 @@ jobs:
name: release
path: release
build-cirrus:
name: "Build using Cirrus CI"
runs-on: "ubuntu-latest"
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Create release directory
run: |
mkdir -p release
- name: Execute Cirrus CI build script
env:
CIRRUS_KEY: ${{ secrets.CIRRUS_TOKEN }}
run: |
python ./deployment/cirrus/build.py "${{ github.ref_name }}" "release/" "${{ inputs.caller }}"
- name: Save release as artifact
uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
with:
retention-days: 3
name: release
path: release
build-deb:
name: "Build Debian installers"
runs-on: "ubuntu-20.04"

View file

@ -50,6 +50,9 @@ jobs:
build-release:
needs: [initialize-release-job]
uses: ./.github/workflows/build_releases.yml
with:
caller: "deployment"
secrets: inherit
generate-choco:
needs: [build-release]

View file

@ -36,6 +36,9 @@ jobs:
build-release:
needs: [initialize-job]
uses: ./.github/workflows/build_releases.yml
with:
caller: "nightly"
secrets: inherit
upload-release:
name: upload-release

View file

@ -1,5 +1,6 @@
{
"MD013": false,
"MD041": false,
"MD033": false,
"MD040": false,
"MD024": false,

View file

@ -284,8 +284,7 @@ You can also try to use the generated release binaries and manually install on y
- [Latest stable release](https://github.com/ClementTsang/bottom/releases/latest), generated off of the release branch
- [Latest nightly release](https://github.com/ClementTsang/bottom/releases/tag/nightly), generated daily off of the master branch at 00:00 UTC
- FreeBSD builds can be found [here](https://api.cirrus-ci.com/v1/artifact/github/ClementTsang/bottom/freebsd_build/binaries.zip)
- macOS ARM builds can be found [here](https://api.cirrus-ci.com/v1/artifact/github/ClementTsang/bottom/macos_build/binaries.zip)
- Note that for now, FreeBSD and ARM macOS builds are primarily only available on the nightly release.
#### Auto-completion

180
deployment/cirrus/build.py Normal file
View file

@ -0,0 +1,180 @@
#!/bin/python3
# A simple script to trigger Cirrus CI builds and get the release artifacts through Cirrus CI's GraphQL interface.
# Expects the API key to be set in CIRRUS_KEY.
import os
import json
import sys
from textwrap import dedent
from time import sleep, time
from pathlib import Path
from typing import Optional
from urllib.request import Request, urlopen, urlretrieve
URL = "https://api.cirrus-ci.com/graphql"
TASKS = [
("freebsd_build", "bottom_x86_64-unknown-freebsd.tar.gz"),
("macos_build", "bottom_aarch64-apple-darwin.tar.gz"),
]
DL_URL_TEMPLATE = "https://api.cirrus-ci.com/v1/artifact/build/%s/%s/binaries/%s"
def make_query_request(key: str, branch: str, build_type: str):
print("Creating query request.")
mutation_id = "Cirrus CI Build {}-{}-{}".format(build_type, branch, int(time()))
query = """
mutation CreateCirrusCIBuild (
$repo: ID!,
$branch: String!,
$mutation_id: String!
) {
createBuild(
input: {
repositoryId: $repo,
branch: $branch,
clientMutationId: $mutation_id,
}
) {
build {
id,
status
}
}
}
"""
params = {
"repo": "6646638922956800",
"branch": branch,
"mutation_id": mutation_id,
}
data = {"query": dedent(query), "variables": params}
data = json.dumps(data).encode()
request = Request(URL, data=data, method="POST")
request.add_header("Authorization", "Bearer {}".format(key))
return request
def check_build_status(key: str, id: str) -> Optional[str]:
query = """
query BuildStatus($id: ID!) {
build(id: $id) {
status
}
}
"""
params = {
"id": id,
}
data = {"query": dedent(query), "variables": params}
data = json.dumps(data).encode()
request = Request(URL, data=data, method="POST")
request.add_header("Authorization", "Bearer {}".format(key))
with urlopen(request) as response:
response = json.load(response)
if response.get("errors") is not None:
print("There was an error in the returned response.")
return None
try:
status = response["data"]["build"]["status"]
return status
except KeyError:
print("There was an issue with creating a build job.")
return None
def try_download(build_id: str, dl_path: Path):
for (task, file) in TASKS:
url = DL_URL_TEMPLATE % (build_id, task, file)
out = dl_path / file
print("Downloading {} to {}".format(file, out))
urlretrieve(url, out)
def main():
args = sys.argv
env = os.environ
key = env["CIRRUS_KEY"]
branch = args[1]
dl_path = args[2] if len(args) >= 3 else ""
dl_path = Path(dl_path)
build_type = args[3] if len(args) >= 4 else "build"
build_id = args[4] if len(args) >= 5 else None
# Check if this build has already been completed before.
if build_id is not None:
print("Previous build ID was provided, checking if complete.")
status = check_build_status(key, build_id)
if status.startswith("COMPLETE"):
print("Starting download of previous build ID")
try_download(build_id, dl_path)
else:
# Try up to three times
MAX_ATTEMPTS = 3
success = False
for i in range(MAX_ATTEMPTS):
if success:
break
print("Attempt {}:".format(i + 1))
with urlopen(make_query_request(key, branch, build_type)) as response:
response = json.load(response)
if response.get("errors") is not None:
print("There was an error in the returned response.")
continue
try:
build_id = response["data"]["createBuild"]["build"]["id"]
print("Created build job {}.".format(build_id))
except KeyError:
print("There was an issue with creating a build job.")
continue
# First, sleep 4 minutes, as it's unlikely it'll finish before then.
SLEEP_MINUTES = 4
print("Sleeping for {} minutes.".format(SLEEP_MINUTES))
sleep(60 * SLEEP_MINUTES)
print("Mandatory nap over. Starting to check for completion.")
MINUTES = 10
SLEEP_SEC = 30
TRIES = int(MINUTES * (60 / SLEEP_SEC)) # Works out to 20 tries.
for attempt in range(TRIES):
print("Checking...")
status = check_build_status(key, build_id)
if status.startswith("COMPLETE"):
print("Build complete. Downloading artifact files.")
sleep(5)
try_download(build_id, dl_path)
success = True
break
else:
print("Build status: {}".format(status or "unknown"))
if status == "ABORTED":
print("Build aborted, bailing.")
break
elif status.lower().startswith("fail"):
print("Build failed, bailing.")
break
elif attempt + 1 < TRIES:
sleep(SLEEP_SEC)
else:
print("Build failed to complete after {} minutes, bailing.".format(MINUTES))
continue
if not success:
exit(2)
if __name__ == "__main__":
main()