From b58d982a31e53d78bf306e5aee37917c25c10c5a Mon Sep 17 00:00:00 2001 From: Clement Tsang <34804052+ClementTsang@users.noreply.github.com> Date: Fri, 2 Aug 2024 04:49:53 +0000 Subject: [PATCH] ci: some more cirrus release tweaks (#1531) * ci: better cirrus release * add cancellation * limit number of concurrent cargo jobs * update test skip, clean up, clean tasks --- .cirrus.yml | 2 +- .gitignore | 5 +- scripts/cirrus/release.py | 127 +++++++++++++++++++++++++++++++++----- 3 files changed, 118 insertions(+), 16 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index e8be3976..060a94ee 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -43,7 +43,7 @@ test_task: auto_cancellation: "false" # We set this to false to prevent nightly builds from affecting this only_if: $CIRRUS_BUILD_SOURCE != "api" && ($CIRRUS_BRANCH == "main" || $CIRRUS_PR != "") timeout_in: "15m" - skip: "!changesInclude('.cargo/**', '.cirrus.yml', 'sample_configs/**', 'src/**', 'tests/**', 'build.rs', 'Cargo.lock', 'Cargo.toml', 'clippy.toml', 'rustfmt.toml')" + skip: "!changesInclude('.cargo/**', 'sample_configs/**', 'scripts/cirrus/**', 'src/**', 'tests/**', '.cirrus.yml', 'build.rs', 'Cargo.lock', 'Cargo.toml', 'clippy.toml', 'rustfmt.toml')" matrix: - name: "FreeBSD 14 Test" freebsd_instance: diff --git a/.gitignore b/.gitignore index 5619dc9b..94e6b7eb 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,7 @@ supply-chain/ # samply profiling profile.json -**/venv/ \ No newline at end of file +**/venv/ + +# Sometimes used for scripts +.ruff_cache diff --git a/scripts/cirrus/release.py b/scripts/cirrus/release.py index ba12d60a..8c131592 100644 --- a/scripts/cirrus/release.py +++ b/scripts/cirrus/release.py @@ -28,9 +28,8 @@ URL = "https://api.cirrus-ci.com/graphql" DL_URL_TEMPLATE = "https://api.cirrus-ci.com/v1/artifact/build/%s/%s/binaries/%s" -def make_query_request(key: str, branch: str, build_type: str): +def make_query_request(key: str, branch: str, mutation_id: str): print("Creating query request.") - mutation_id = "Cirrus CI Build {}-{}-{}".format(build_type, branch, int(time())) # Dumb but if it works... config_override = ( @@ -38,6 +37,7 @@ def make_query_request(key: str, branch: str, build_type: str): .read_text() .replace("# -PLACEHOLDER FOR CI-", 'BTM_BUILD_RELEASE_CALLER: "ci"') ) + query = """ mutation CreateCirrusCIBuild ( $repo: ID!, @@ -60,12 +60,14 @@ def make_query_request(key: str, branch: str, build_type: str): } } """ + params = { "repo": "6646638922956800", "branch": branch, "mutation_id": mutation_id, "config_override": dedent(config_override), } + data = {"query": dedent(query), "variables": params} data = json.dumps(data).encode() @@ -75,16 +77,17 @@ def make_query_request(key: str, branch: str, build_type: str): return request -def check_build_status(key: str, id: str) -> Optional[str]: +def check_build_status(key: str, build_id: str) -> Optional[str]: query = """ query BuildStatus($id: ID!) { build(id: $id) { status + } } - } """ + params = { - "id": id, + "id": build_id, } data = {"query": dedent(query), "variables": params} @@ -102,10 +105,80 @@ def check_build_status(key: str, id: str) -> Optional[str]: status = response["data"]["build"]["status"] return status except KeyError: - print("There was an issue with creating a build job.") + print("There was an issue with checking the build status.") return None +def check_build_tasks(key: str, build_id: str) -> Optional[List[str]]: + query = """ + query Build($id:ID!) { + build(id:$id){ + tasks { + id + } + } + } + """ + + params = { + "id": build_id, + } + + data = {"query": dedent(query), "variables": params} + data = json.dumps(data).encode() + + request = Request(URL, data=data, method="POST") + request.add_header("Authorization", "Bearer {}".format(key)) + with urlopen(request) as response: + response = json.load(response) + + if response.get("errors") is not None: + print("There was an error in the returned response.") + return None + + try: + tasks = [task["id"] for task in response["data"]["build"]["tasks"]] + return tasks + except KeyError: + print("There was an issue with getting the list of task ids.") + return None + + +def stop_build_tasks(key: str, task_ids: List[str], mutation_id: str) -> bool: + query = """ + mutation StopCirrusCiTasks ( + $task_ids: [ID!]!, + $mutation_id: String!, + ) { + batchAbort ( + input: { + taskIds: $task_ids, + clientMutationId: $mutation_id + } + ) { + tasks { + id + } + } + } + """ + + params = { + "task_ids": task_ids, + "mutation_id": mutation_id, + } + + data = {"query": dedent(query), "variables": params} + data = json.dumps(data).encode() + + request = Request(URL, data=data, method="POST") + request.add_header("Authorization", "Bearer {}".format(key)) + + with urlopen(request) as response: + response = json.load(response) + return len(response["data"]["batchAbort"]["tasks"]) == len(task_ids) + + def try_download(build_id: str, dl_path: Path): for task, file in TASKS: url = DL_URL_TEMPLATE % (build_id, task, file) @@ -138,17 +211,36 @@ def main(): # Try up to three times MAX_ATTEMPTS = 5 success = False + tasks = [] + mutation_id = None for i in range(MAX_ATTEMPTS): if success: break + print(f"Attempt {i + 1}:") - with urlopen(make_query_request(key, branch, build_type)) as response: - response = json.load(response) + if tasks and mutation_id: + print("Killing previous tasks first...") - if response.get("errors") is not None: - print("There was an error in the returned response.") + if stop_build_tasks(key, tasks, mutation_id): + print("All previous tasks successfully stopped.") + else: + print( + "Not all previous tasks stopped. This isn't a problem but it is a waste." + ) + + tasks = [] + mutation_id = "Cirrus CI Build {}-{}-{}".format( + build_type, branch, int(time()) + ) + + with urlopen(make_query_request(key, branch, mutation_id)) as response: + response = json.load(response) + errors = response.get("errors") + + if errors is not None: + print(f"There was an error in the returned response: {str(errors)}") continue try: @@ -158,16 +250,22 @@ def main(): print("There was an issue with creating a build job.") continue - # First, sleep 4 minutes, as it's unlikely it'll finish before then. + # First, sleep X minutes total, as it's unlikely it'll finish before then. SLEEP_MINUTES = 4 print(f"Sleeping for {SLEEP_MINUTES} minutes.") - sleep(60 * SLEEP_MINUTES) - print("Mandatory nap over. Starting to check for completion.") + + # Sleep and check for tasks out every 10 seconds + for _ in range(SLEEP_MINUTES * 6): + sleep(10) + if not tasks: + tasks = check_build_tasks(key, build_id) MINUTES = 10 SLEEP_SEC = 30 TRIES = int(MINUTES * (60 / SLEEP_SEC)) # Works out to 20 tries. + print(f"Mandatory nap over. Checking for completion for {MINUTES} min.") + for attempt in range(TRIES): print("Checking...") try: @@ -179,7 +277,8 @@ def main(): success = True break else: - print("Build status: {}".format(status or "unknown")) + print(f"Build status: {(status or 'unknown')}") + if status == "ABORTED": print("Build aborted, bailing.") break