From f15e026101a99487fd1debb87e6c879eb9b215c1 Mon Sep 17 00:00:00 2001 From: Raghul Nanth A <61490162+NanthR@users.noreply.github.com> Date: Mon, 12 Jun 2023 01:18:24 +0530 Subject: [PATCH 1/5] ci(metrics): Run measurement functions in parallel feat(xtask): Split metrics function --- .github/workflows/metrics.yaml | 234 +++++++++++++++++++++++++++++++-- xtask/src/flags.rs | 29 +++- xtask/src/metrics.rs | 76 ++++++----- 3 files changed, 292 insertions(+), 47 deletions(-) diff --git a/.github/workflows/metrics.yaml b/.github/workflows/metrics.yaml index 3fe2fc917a..233af42f42 100644 --- a/.github/workflows/metrics.yaml +++ b/.github/workflows/metrics.yaml @@ -11,20 +11,234 @@ env: RUSTUP_MAX_RETRIES: 10 jobs: - metrics: - if: github.repository == 'rust-lang/rust-analyzer' + setup_cargo: runs-on: ubuntu-latest + steps: + - name: Install Rust toolchain + run: | + rustup update --no-self-update stable + rustup component add rustfmt rust-src + - name: Cache cargo + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-cargo-${{ github.sha }} + + build_metrics: + runs-on: ubuntu-latest + needs: setup_cargo steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Install Rust toolchain - run: | - rustup update --no-self-update stable - rustup component add rustfmt rust-src + - name: Restore cargo cache + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-cargo-${{ github.sha }} - - name: Collect metrics - run: cargo xtask metrics - env: - METRICS_TOKEN: ${{ secrets.METRICS_TOKEN }} + + - name: Collect build metrics + run: cargo xtask metrics build + + - name: Cache target + uses: actions/cache@v3 + with: + path: target/ + key: ${{ runner.os }}-target-${{ github.sha }} + + - name: Upload build metrics + uses: actions/upload-artifact@v3 + with: + name: build-${{ github.sha }} + path: target/build.json + if-no-files-found: error + + self_metrics: + runs-on: ubuntu-latest + needs: [setup_cargo, build_metrics] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Restore cargo cache + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-cargo-${{ github.sha }} + + - name: Restore target cache + uses: actions/cache@v3 + with: + path: target/ + key: ${{ runner.os }}-target-${{ github.sha }} + + - name: Collect build metrics + run: cargo xtask metrics self + + - name: Upload build metrics + uses: actions/upload-artifact@v3 + with: + name: self-${{ github.sha }} + path: target/self.json + if-no-files-found: error + + ripgrep_metrics: + runs-on: ubuntu-latest + needs: [setup_cargo, build_metrics] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Restore cargo cache + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-cargo-${{ github.sha }} + + - name: Restore target cache + uses: actions/cache@v3 + with: + path: target/ + key: ${{ runner.os }}-target-${{ github.sha }} + + - name: Collect build metrics + run: cargo xtask metrics ripgrep + + - name: Upload ripgrep metrics + uses: actions/upload-artifact@v3 + with: + name: ripgrep-${{ github.sha }} + path: target/ripgrep.json + if-no-files-found: error + + webrender_metrics: + runs-on: ubuntu-latest + needs: [setup_cargo, build_metrics] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Restore cargo cache + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-cargo-${{ github.sha }} + + - name: Restore target cache + uses: actions/cache@v3 + with: + path: target/ + key: ${{ runner.os }}-target-${{ github.sha }} + + - name: Collect webrender metrics + run: cargo xtask metrics webrender + + - name: Upload webrender metrics + uses: actions/upload-artifact@v3 + with: + name: webrender-${{ github.sha }} + path: target/webrender.json + if-no-files-found: error + + diesel_metrics: + runs-on: ubuntu-latest + needs: [setup_cargo, build_metrics] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Restore cargo cache + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-cargo-${{ github.sha }} + + - name: Restore target cache + uses: actions/cache@v3 + with: + path: target/ + key: ${{ runner.os }}-target-${{ github.sha }} + + - name: Collect build metrics + run: cargo xtask metrics diesel + + - name: Upload build metrics + uses: actions/upload-artifact@v3 + with: + name: diesel-${{ github.sha }} + path: target/diesel.json + if-no-files-found: error + + + + generate_final_metrics: + runs-on: ubuntu-latest + needs: [build_metrics, self_metrics, ripgrep_metrics, webrender_metrics, diesel_metrics] + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Download build metrics + uses: actions/download-artifact@v3 + with: + name: build-${{ github.sha }} + + - name: Download self metrics + uses: actions/download-artifact@v3 + with: + name: self-${{ github.sha }} + + - name: Download ripgrep metrics + uses: actions/download-artifact@v3 + with: + name: ripgrep-${{ github.sha }} + + - name: Download webrender metrics + uses: actions/download-artifact@v3 + with: + name: webrender-${{ github.sha }} + + - name: Download diesel metrics + uses: actions/download-artifact@v3 + with: + name: diesel-${{ github.sha }} + + - name: Combine json + run: | + git clone --depth 1 https://$METRICS_TOKEN@github.com/nanthR/metrics.git + jq -s ".[0] * .[1] * .[2] * .[3] * .[4]" build.json self.json ripgrep.json webrender.json diesel.json -c >> metrics/metrics.json + git -C metrics add . + git -C metrics -c user.name=Bot -c user.email=dummy@example.com commit --message 📈 + git -C metrics push origin master + env: + METRICS_TOKEN: ${{ secrets.METRICS_TOKEN }} diff --git a/xtask/src/flags.rs b/xtask/src/flags.rs index 2100479701..47d0ea254b 100644 --- a/xtask/src/flags.rs +++ b/xtask/src/flags.rs @@ -1,5 +1,7 @@ #![allow(unreachable_pub)] +use std::str::FromStr; + use crate::install::{ClientOpt, Malloc, ServerOpt}; xflags::xflags! { @@ -42,7 +44,7 @@ xflags::xflags! { required changelog: String } cmd metrics { - optional --dry-run + optional measurement_type: MeasurementType } /// Builds a benchmark version of rust-analyzer and puts it into `./target`. cmd bb { @@ -105,9 +107,32 @@ pub struct PublishReleaseNotes { pub dry_run: bool, } +#[derive(Debug)] +pub enum MeasurementType { + Build, + AnalyseSelf, + AnalyseRipgrep, + AnalyseWebRender, + AnalyseDiesel, +} + +impl FromStr for MeasurementType { + type Err = String; + fn from_str(s: &str) -> Result { + match s { + "build" => Ok(Self::Build), + "self" => Ok(Self::AnalyseSelf), + "ripgrep" => Ok(Self::AnalyseRipgrep), + "webrender" => Ok(Self::AnalyseWebRender), + "diesel" => Ok(Self::AnalyseDiesel), + _ => Err("Invalid option".to_string()), + } + } +} + #[derive(Debug)] pub struct Metrics { - pub dry_run: bool, + pub measurement_type: Option, } #[derive(Debug)] diff --git a/xtask/src/metrics.rs b/xtask/src/metrics.rs index b6f730dbf1..d51dfb183f 100644 --- a/xtask/src/metrics.rs +++ b/xtask/src/metrics.rs @@ -1,6 +1,6 @@ use std::{ collections::BTreeMap, - env, fs, + fs, io::Write as _, path::Path, time::{Instant, SystemTime, UNIX_EPOCH}, @@ -9,16 +9,13 @@ use std::{ use anyhow::{bail, format_err}; use xshell::{cmd, Shell}; -use crate::flags; +use crate::flags::{self, MeasurementType}; type Unit = String; impl flags::Metrics { pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> { let mut metrics = Metrics::new(sh)?; - if !self.dry_run { - sh.remove_path("./target/release")?; - } if !Path::new("./target/rustc-perf").exists() { sh.create_dir("./target/rustc-perf")?; cmd!(sh, "git clone https://github.com/rust-lang/rustc-perf.git ./target/rustc-perf") @@ -32,38 +29,47 @@ impl flags::Metrics { let _env = sh.push_env("RA_METRICS", "1"); - { - // https://github.com/rust-lang/rust-analyzer/issues/9997 - let _d = sh.push_dir("target/rustc-perf/collector/benchmarks/webrender"); - cmd!(sh, "cargo update -p url --precise 1.6.1").run()?; - } - metrics.measure_build(sh)?; - metrics.measure_analysis_stats_self(sh)?; - metrics.measure_analysis_stats(sh, "ripgrep")?; - metrics.measure_analysis_stats(sh, "webrender")?; - metrics.measure_analysis_stats(sh, "diesel/diesel")?; - - if !self.dry_run { - let _d = sh.push_dir("target"); - let metrics_token = env::var("METRICS_TOKEN").unwrap(); - cmd!( - sh, - "git clone --depth 1 https://{metrics_token}@github.com/rust-analyzer/metrics.git" - ) - .run()?; - - { - let mut file = - fs::File::options().append(true).open("target/metrics/metrics.json")?; - writeln!(file, "{}", metrics.json())?; + let filename = match self.measurement_type { + Some(ms) => match ms { + MeasurementType::Build => { + metrics.measure_build(sh)?; + "build.json" + } + MeasurementType::AnalyseSelf => { + metrics.measure_analysis_stats_self(sh)?; + "self.json" + } + MeasurementType::AnalyseRipgrep => { + metrics.measure_analysis_stats(sh, "ripgrep")?; + "ripgrep.json" + } + MeasurementType::AnalyseWebRender => { + { + // https://github.com/rust-lang/rust-analyzer/issues/9997 + let _d = sh.push_dir("target/rustc-perf/collector/benchmarks/webrender"); + cmd!(sh, "cargo update -p url --precise 1.6.1").run()?; + } + metrics.measure_analysis_stats(sh, "webrender")?; + "webrender.json" + } + MeasurementType::AnalyseDiesel => { + metrics.measure_analysis_stats(sh, "diesel/diesel")?; + "diesel.json" + } + }, + None => { + metrics.measure_build(sh)?; + metrics.measure_analysis_stats_self(sh)?; + metrics.measure_analysis_stats(sh, "ripgrep")?; + metrics.measure_analysis_stats(sh, "webrender")?; + metrics.measure_analysis_stats(sh, "diesel/diesel")?; + "all.json" } + }; - let _d = sh.push_dir("metrics"); - cmd!(sh, "git add .").run()?; - cmd!(sh, "git -c user.name=Bot -c user.email=dummy@example.com commit --message 📈") - .run()?; - cmd!(sh, "git push origin master").run()?; - } + let mut file = + fs::File::options().write(true).create(true).open(format!("target/{}", filename))?; + writeln!(file, "{}", metrics.json())?; eprintln!("{metrics:#?}"); Ok(()) } From 5d77839b25963ce9bb06d8e1de760106477e6fe1 Mon Sep 17 00:00:00 2001 From: Raghul Nanth A <61490162+NanthR@users.noreply.github.com> Date: Mon, 12 Jun 2023 23:14:04 +0530 Subject: [PATCH 2/5] style(metrics): Change spelling --- xtask/src/flags.rs | 16 ++++++++-------- xtask/src/metrics.rs | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/xtask/src/flags.rs b/xtask/src/flags.rs index 47d0ea254b..7720ad69fe 100644 --- a/xtask/src/flags.rs +++ b/xtask/src/flags.rs @@ -110,10 +110,10 @@ pub struct PublishReleaseNotes { #[derive(Debug)] pub enum MeasurementType { Build, - AnalyseSelf, - AnalyseRipgrep, - AnalyseWebRender, - AnalyseDiesel, + AnalyzeSelf, + AnalyzeRipgrep, + AnalyzeWebRender, + AnalyzeDiesel, } impl FromStr for MeasurementType { @@ -121,10 +121,10 @@ impl FromStr for MeasurementType { fn from_str(s: &str) -> Result { match s { "build" => Ok(Self::Build), - "self" => Ok(Self::AnalyseSelf), - "ripgrep" => Ok(Self::AnalyseRipgrep), - "webrender" => Ok(Self::AnalyseWebRender), - "diesel" => Ok(Self::AnalyseDiesel), + "self" => Ok(Self::AnalyzeSelf), + "ripgrep" => Ok(Self::AnalyzeRipgrep), + "webrender" => Ok(Self::AnalyzeWebRender), + "diesel" => Ok(Self::AnalyzeDiesel), _ => Err("Invalid option".to_string()), } } diff --git a/xtask/src/metrics.rs b/xtask/src/metrics.rs index d51dfb183f..6853742319 100644 --- a/xtask/src/metrics.rs +++ b/xtask/src/metrics.rs @@ -35,15 +35,15 @@ impl flags::Metrics { metrics.measure_build(sh)?; "build.json" } - MeasurementType::AnalyseSelf => { + MeasurementType::AnalyzeSelf => { metrics.measure_analysis_stats_self(sh)?; "self.json" } - MeasurementType::AnalyseRipgrep => { + MeasurementType::AnalyzeRipgrep => { metrics.measure_analysis_stats(sh, "ripgrep")?; "ripgrep.json" } - MeasurementType::AnalyseWebRender => { + MeasurementType::AnalyzeWebRender => { { // https://github.com/rust-lang/rust-analyzer/issues/9997 let _d = sh.push_dir("target/rustc-perf/collector/benchmarks/webrender"); @@ -52,7 +52,7 @@ impl flags::Metrics { metrics.measure_analysis_stats(sh, "webrender")?; "webrender.json" } - MeasurementType::AnalyseDiesel => { + MeasurementType::AnalyzeDiesel => { metrics.measure_analysis_stats(sh, "diesel/diesel")?; "diesel.json" } From 68e64025fd0bc455af78d46e83952427499f3a28 Mon Sep 17 00:00:00 2001 From: Raghul Nanth A <61490162+NanthR@users.noreply.github.com> Date: Sat, 24 Jun 2023 22:33:33 +0530 Subject: [PATCH 3/5] fix(name): Push to rust-analyzer metrics repo --- .github/workflows/metrics.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/metrics.yaml b/.github/workflows/metrics.yaml index 233af42f42..ca50980981 100644 --- a/.github/workflows/metrics.yaml +++ b/.github/workflows/metrics.yaml @@ -235,10 +235,11 @@ jobs: - name: Combine json run: | - git clone --depth 1 https://$METRICS_TOKEN@github.com/nanthR/metrics.git + git clone --depth 1 https://$METRICS_TOKEN@github.com/rust-analyzer/metrics.git jq -s ".[0] * .[1] * .[2] * .[3] * .[4]" build.json self.json ripgrep.json webrender.json diesel.json -c >> metrics/metrics.json - git -C metrics add . - git -C metrics -c user.name=Bot -c user.email=dummy@example.com commit --message 📈 - git -C metrics push origin master + cd metrics + git add . + git -c user.name=Bot -c user.email=dummy@example.com commit --message 📈 + git push origin new env: METRICS_TOKEN: ${{ secrets.METRICS_TOKEN }} From 506153b9f3f33f17b601266de1999d3413516890 Mon Sep 17 00:00:00 2001 From: Raghul Nanth A <61490162+NanthR@users.noreply.github.com> Date: Sat, 24 Jun 2023 22:52:04 +0530 Subject: [PATCH 4/5] refactor(workflow): Use build matrix --- .github/workflows/metrics.yaml | 121 +++------------------------------ 1 file changed, 10 insertions(+), 111 deletions(-) diff --git a/.github/workflows/metrics.yaml b/.github/workflows/metrics.yaml index ca50980981..216c218146 100644 --- a/.github/workflows/metrics.yaml +++ b/.github/workflows/metrics.yaml @@ -63,7 +63,10 @@ jobs: path: target/build.json if-no-files-found: error - self_metrics: + other_metrics: + strategy: + matrix: + names: [self, ripgrep, webrender, diesel] runs-on: ubuntu-latest needs: [setup_cargo, build_metrics] @@ -87,123 +90,19 @@ jobs: path: target/ key: ${{ runner.os }}-target-${{ github.sha }} - - name: Collect build metrics - run: cargo xtask metrics self + - name: Collect metrics + run: cargo xtask metrics ${{ matrix.names }} - - name: Upload build metrics + - name: Upload metrics uses: actions/upload-artifact@v3 with: - name: self-${{ github.sha }} - path: target/self.json + name: ${{ matrix.names }}-${{ github.sha }} + path: target/${{ matrix.names }}.json if-no-files-found: error - ripgrep_metrics: - runs-on: ubuntu-latest - needs: [setup_cargo, build_metrics] - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Restore cargo cache - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ github.sha }} - - - name: Restore target cache - uses: actions/cache@v3 - with: - path: target/ - key: ${{ runner.os }}-target-${{ github.sha }} - - - name: Collect build metrics - run: cargo xtask metrics ripgrep - - - name: Upload ripgrep metrics - uses: actions/upload-artifact@v3 - with: - name: ripgrep-${{ github.sha }} - path: target/ripgrep.json - if-no-files-found: error - - webrender_metrics: - runs-on: ubuntu-latest - needs: [setup_cargo, build_metrics] - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Restore cargo cache - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ github.sha }} - - - name: Restore target cache - uses: actions/cache@v3 - with: - path: target/ - key: ${{ runner.os }}-target-${{ github.sha }} - - - name: Collect webrender metrics - run: cargo xtask metrics webrender - - - name: Upload webrender metrics - uses: actions/upload-artifact@v3 - with: - name: webrender-${{ github.sha }} - path: target/webrender.json - if-no-files-found: error - - diesel_metrics: - runs-on: ubuntu-latest - needs: [setup_cargo, build_metrics] - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Restore cargo cache - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ github.sha }} - - - name: Restore target cache - uses: actions/cache@v3 - with: - path: target/ - key: ${{ runner.os }}-target-${{ github.sha }} - - - name: Collect build metrics - run: cargo xtask metrics diesel - - - name: Upload build metrics - uses: actions/upload-artifact@v3 - with: - name: diesel-${{ github.sha }} - path: target/diesel.json - if-no-files-found: error - - - generate_final_metrics: runs-on: ubuntu-latest - needs: [build_metrics, self_metrics, ripgrep_metrics, webrender_metrics, diesel_metrics] + needs: [build_metrics, other_metrics] steps: - name: Checkout repository uses: actions/checkout@v3 From cf34df020d7148eef5b49fb3218ca45a1cbaaa95 Mon Sep 17 00:00:00 2001 From: Raghul Nanth A <61490162+NanthR@users.noreply.github.com> Date: Wed, 28 Jun 2023 18:13:24 +0530 Subject: [PATCH 5/5] fix(metrics): Add repo check --- .github/workflows/metrics.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/metrics.yaml b/.github/workflows/metrics.yaml index 216c218146..37603d6ff6 100644 --- a/.github/workflows/metrics.yaml +++ b/.github/workflows/metrics.yaml @@ -12,6 +12,7 @@ env: jobs: setup_cargo: + if: github.repository == 'rust-lang/rust-analyzer' runs-on: ubuntu-latest steps: - name: Install Rust toolchain