ci: automatically create and get build artifacts from Cirrus CI (#854)

This automatically triggers and grabs the build artifacts for systems that are only supported on Cirrus CI (as of now, FreeBSD and M1 macOS).

* ci: add cirrus build trigger script

* ci: modify build scripts to include cirrus build

* fix some stuff

* update docs

* more fixes

.cirrus.yml

@@ -1,3 +1,5 @@
+%YAML 1.1
+---
 # Configuration for CirrusCI. This is primarily used for
 # FreeBSD and macOS M1 tests and builds.
 
@@ -35,7 +37,8 @@ env:
   CARGO_HUSKY_DONT_INSTALL_HOOKS: true
 
 test_task:
-  only_if: $CIRRUS_CRON == "" && ($CIRRUS_BRANCH == "master" || $CIRRUS_PR != "")
+  auto_cancellation: false
+  only_if: $CIRRUS_BUILD_SOURCE != "api" && ($CIRRUS_BRANCH == "master" || $CIRRUS_PR != "")
   matrix:
     - name: "FreeBSD 13 Test"
       freebsd_instance:
@@ -60,7 +63,8 @@ test_task:
   <<: *CLEANUP_TEMPLATE
 
 build_task:
-  only_if: $CIRRUS_RELEASE != "" || $CIRRUS_CRON == "nightly" || $CIRRUS_API_CREATED == true || $CIRRUS_BRANCH == "master"
+  auto_cancellation: false
+  only_if: $CIRRUS_BUILD_SOURCE == "api"
   env:
     BTM_GENERATE: true
     COMPLETION_DIR: "target/tmp/bottom/completion/"

.github/workflows/build_releases.yml

@@ -8,6 +8,12 @@ name: "Build Releases"
 on:
   workflow_dispatch:
   workflow_call:
+    inputs:
+      caller:
+        description: "The calling workflow."
+        default: ""
+        required: false
+        type: string
 
 env:
   CARGO_INCREMENTAL: 0
@@ -249,6 +255,32 @@ jobs:
           name: release
           path: release
 
+  build-cirrus:
+    name: "Build using Cirrus CI"
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      - name: Create release directory
+        run: |
+          mkdir -p release
+
+      - name: Execute Cirrus CI build script
+        env:
+          CIRRUS_KEY: ${{ secrets.CIRRUS_TOKEN }}
+        run: |
+          python ./deployment/cirrus/build.py "${{ github.ref_name }}" "release/" "${{ inputs.caller }}"
+
+      - name: Save release as artifact
+        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+        with:
+          retention-days: 3
+          name: release
+          path: release
+
   build-deb:
     name: "Build Debian installers"
     runs-on: "ubuntu-20.04"

.github/workflows/deployment.yml

@@ -50,6 +50,9 @@ jobs:
   build-release:
     needs: [initialize-release-job]
     uses: ./.github/workflows/build_releases.yml
+    with:
+      caller: "deployment"
+    secrets: inherit
 
   generate-choco:
     needs: [build-release]

.github/workflows/nightly.yml

@@ -36,6 +36,9 @@ jobs:
   build-release:
     needs: [initialize-job]
     uses: ./.github/workflows/build_releases.yml
+    with:
+      caller: "nightly"
+    secrets: inherit
 
   upload-release:
     name: upload-release

.markdownlint.json

@@ -1,5 +1,6 @@
 {
   "MD013": false,
+  "MD041": false,
   "MD033": false,
   "MD040": false,
   "MD024": false,

README.md

@@ -284,8 +284,7 @@ You can also try to use the generated release binaries and manually install on y
 
 - [Latest stable release](https://github.com/ClementTsang/bottom/releases/latest), generated off of the release branch
 - [Latest nightly release](https://github.com/ClementTsang/bottom/releases/tag/nightly), generated daily off of the master branch at 00:00 UTC
-  - FreeBSD builds can be found [here](https://api.cirrus-ci.com/v1/artifact/github/ClementTsang/bottom/freebsd_build/binaries.zip)
-  - macOS ARM builds can be found [here](https://api.cirrus-ci.com/v1/artifact/github/ClementTsang/bottom/macos_build/binaries.zip)
+  - Note that for now, FreeBSD and ARM macOS builds are primarily only available on the nightly release.
 
 #### Auto-completion
 

deployment/cirrus/build.py

@@ -0,0 +1,180 @@
+#!/bin/python3
+
+# A simple script to trigger Cirrus CI builds and get the release artifacts through Cirrus CI's GraphQL interface.
+# Expects the API key to be set in CIRRUS_KEY.
+
+import os
+import json
+import sys
+from textwrap import dedent
+from time import sleep, time
+from pathlib import Path
+from typing import Optional
+
+from urllib.request import Request, urlopen, urlretrieve
+
+URL = "https://api.cirrus-ci.com/graphql"
+TASKS = [
+    ("freebsd_build", "bottom_x86_64-unknown-freebsd.tar.gz"),
+    ("macos_build", "bottom_aarch64-apple-darwin.tar.gz"),
+]
+DL_URL_TEMPLATE = "https://api.cirrus-ci.com/v1/artifact/build/%s/%s/binaries/%s"
+
+
+def make_query_request(key: str, branch: str, build_type: str):
+    print("Creating query request.")
+    mutation_id = "Cirrus CI Build {}-{}-{}".format(build_type, branch, int(time()))
+    query = """
+        mutation CreateCirrusCIBuild (
+            $repo: ID!,
+            $branch: String!,
+            $mutation_id: String!
+        ) {
+            createBuild(
+                input: {
+                    repositoryId: $repo,
+                    branch: $branch,
+                    clientMutationId: $mutation_id,
+                }
+            ) {
+                build {
+                    id,
+                    status
+                }
+            }
+        }
+    """
+    params = {
+        "repo": "6646638922956800",
+        "branch": branch,
+        "mutation_id": mutation_id,
+    }
+    data = {"query": dedent(query), "variables": params}
+    data = json.dumps(data).encode()
+
+    request = Request(URL, data=data, method="POST")
+    request.add_header("Authorization", "Bearer {}".format(key))
+
+    return request
+
+
+def check_build_status(key: str, id: str) -> Optional[str]:
+    query = """
+        query BuildStatus($id: ID!) {
+            build(id: $id) {
+                status
+        }
+    }
+    """
+    params = {
+        "id": id,
+    }
+
+    data = {"query": dedent(query), "variables": params}
+    data = json.dumps(data).encode()
+
+    request = Request(URL, data=data, method="POST")
+    request.add_header("Authorization", "Bearer {}".format(key))
+    with urlopen(request) as response:
+        response = json.load(response)
+        if response.get("errors") is not None:
+            print("There was an error in the returned response.")
+            return None
+
+        try:
+            status = response["data"]["build"]["status"]
+            return status
+        except KeyError:
+            print("There was an issue with creating a build job.")
+            return None
+
+
+def try_download(build_id: str, dl_path: Path):
+    for (task, file) in TASKS:
+        url = DL_URL_TEMPLATE % (build_id, task, file)
+        out = dl_path / file
+        print("Downloading {} to {}".format(file, out))
+        urlretrieve(url, out)
+
+
+def main():
+    args = sys.argv
+    env = os.environ
+
+    key = env["CIRRUS_KEY"]
+    branch = args[1]
+    dl_path = args[2] if len(args) >= 3 else ""
+    dl_path = Path(dl_path)
+    build_type = args[3] if len(args) >= 4 else "build"
+    build_id = args[4] if len(args) >= 5 else None
+
+    # Check if this build has already been completed before.
+    if build_id is not None:
+        print("Previous build ID was provided, checking if complete.")
+        status = check_build_status(key, build_id)
+        if status.startswith("COMPLETE"):
+            print("Starting download of previous build ID")
+            try_download(build_id, dl_path)
+    else:
+        # Try up to three times
+        MAX_ATTEMPTS = 3
+        success = False
+
+        for i in range(MAX_ATTEMPTS):
+            if success:
+                break
+            print("Attempt {}:".format(i + 1))
+
+            with urlopen(make_query_request(key, branch, build_type)) as response:
+                response = json.load(response)
+
+                if response.get("errors") is not None:
+                    print("There was an error in the returned response.")
+                    continue
+
+                try:
+                    build_id = response["data"]["createBuild"]["build"]["id"]
+                    print("Created build job {}.".format(build_id))
+                except KeyError:
+                    print("There was an issue with creating a build job.")
+                    continue
+
+                # First, sleep 4 minutes, as it's unlikely it'll finish before then.
+                SLEEP_MINUTES = 4
+                print("Sleeping for {} minutes.".format(SLEEP_MINUTES))
+                sleep(60 * SLEEP_MINUTES)
+                print("Mandatory nap over. Starting to check for completion.")
+
+                MINUTES = 10
+                SLEEP_SEC = 30
+                TRIES = int(MINUTES * (60 / SLEEP_SEC))  # Works out to 20 tries.
+
+                for attempt in range(TRIES):
+                    print("Checking...")
+                    status = check_build_status(key, build_id)
+                    if status.startswith("COMPLETE"):
+                        print("Build complete. Downloading artifact files.")
+                        sleep(5)
+                        try_download(build_id, dl_path)
+                        success = True
+                        break
+                    else:
+                        print("Build status: {}".format(status or "unknown"))
+                        if status == "ABORTED":
+                            print("Build aborted, bailing.")
+                            break
+                        elif status.lower().startswith("fail"):
+                            print("Build failed, bailing.")
+                            break
+                        elif attempt + 1 < TRIES:
+                            sleep(SLEEP_SEC)
+                else:
+                    print("Build failed to complete after {} minutes, bailing.".format(MINUTES))
+                    continue
+
+        if not success:
+            exit(2)
+
+
+if __name__ == "__main__":
+    main()