From f92bfb580780cda02f9ba8a935538f984d8a4c0d Mon Sep 17 00:00:00 2001 From: Veetaha Date: Sun, 21 Jun 2020 15:58:34 +0300 Subject: [PATCH] Gzip artifacts Co-authored-by: bjorn3 Override miniz_oxide to build it with optimizations Building this crate with optimizations decreases the gzipping part of `cargo xtask dist` from `30-40s` down to `3s`, the overhead for `rustc` to apply optimizations is miserable on this background --- Cargo.lock | 22 ++++++++++++++++++++++ Cargo.toml | 5 +++++ editors/code/src/main.ts | 17 +++++++++-------- editors/code/src/net.ts | 16 ++++++++-------- xtask/Cargo.toml | 1 + xtask/src/dist.rs | 31 +++++++++++++++++++++++-------- 6 files changed, 68 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 916fc53e07..108403ce63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -207,6 +207,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "crc32fast" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.4.2" @@ -340,6 +349,18 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" +[[package]] +name = "flate2" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68c90b0fc46cf89d227cc78b40e494ff81287a92dd07631e5af0d06fe3cf885e" +dependencies = [ + "cfg-if", + "crc32fast", + "libc", + "miniz_oxide", +] + [[package]] name = "flycheck" version = "0.1.0" @@ -1993,6 +2014,7 @@ name = "xtask" version = "0.1.0" dependencies = [ "anyhow", + "flate2", "pico-args", "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 189f4ea2bd..612e6809f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,11 @@ opt-level = 0 [profile.release.package.xtask] opt-level = 0 +# Gzipping the artifacts is up to 10 times faster with optimizations (`cargo xtask dist`). +# `miniz_oxide` is the direct dependency of `flate2` which does all the heavy lifting +[profile.dev.package.miniz_oxide] +opt-level = 3 + [patch.'crates-io'] # rowan = { path = "../rowan" } diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts index eda95ae5c4..bd99d696ad 100644 --- a/editors/code/src/main.ts +++ b/editors/code/src/main.ts @@ -274,13 +274,13 @@ async function getServer(config: Config, state: PersistentState): Promise true, () => false); if (!exists) { await state.updateServerVersion(undefined); @@ -309,7 +309,7 @@ async function getServer(config: Config, state: PersistentState): Promise artifact.name === binaryName); + const artifact = release.assets.find(artifact => artifact.name === `rust-analyzer-${platform}.gz`); assert(!!artifact, `Bad release: ${JSON.stringify(release)}`); // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error. @@ -321,6 +321,7 @@ async function getServer(config: Config, state: PersistentState): Promise { let lastPercentage = 0; - await downloadFile(opts.url, tempFile, opts.mode, (readBytes, totalBytes) => { + await downloadFile(opts.url, tempFile, opts.mode, !!opts.gunzip, (readBytes, totalBytes) => { const newPercentage = (readBytes / totalBytes) * 100; progress.report({ message: newPercentage.toFixed(0) + "%", @@ -97,16 +99,11 @@ export async function download(opts: DownloadOpts) { await fs.promises.rename(tempFile, opts.dest); } -/** - * Downloads file from `url` and stores it at `destFilePath` with `mode` (unix permissions). - * `onProgress` callback is called on recieveing each chunk of bytes - * to track the progress of downloading, it gets the already read and total - * amount of bytes to read as its parameters. - */ async function downloadFile( url: string, destFilePath: fs.PathLike, mode: number | undefined, + gunzip: boolean, onProgress: (readBytes: number, totalBytes: number) => void ): Promise { const res = await fetch(url); @@ -130,7 +127,10 @@ async function downloadFile( }); const destFileStream = fs.createWriteStream(destFilePath, { mode }); - await pipeline(res.body, destFileStream); + const srcStream = gunzip ? res.body.pipe(zlib.createGunzip()) : res.body; + + await pipeline(srcStream, destFileStream); + await new Promise(resolve => { destFileStream.on("close", resolve); destFileStream.destroy(); diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index a8b9b010d7..8045a98eaf 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -14,3 +14,4 @@ pico-args = "0.3.1" quote = "1.0.2" proc-macro2 = "1.0.8" anyhow = "1.0.26" +flate2 = "1.0" diff --git a/xtask/src/dist.rs b/xtask/src/dist.rs index aef68089e1..b8f68027c0 100644 --- a/xtask/src/dist.rs +++ b/xtask/src/dist.rs @@ -1,4 +1,10 @@ -use std::path::PathBuf; +use flate2::{write::GzEncoder, Compression}; +use std::{ + env, + fs::File, + io, + path::{Path, PathBuf}, +}; use anyhow::Result; @@ -16,7 +22,7 @@ pub fn run_dist(nightly: bool, client_version: Option) -> Result<()> { let release_tag = if nightly { "nightly".to_string() } else { date_iso()? }; dist_client(&version, &release_tag)?; } - dist_server(nightly)?; + dist_server()?; Ok(()) } @@ -46,17 +52,14 @@ fn dist_client(version: &str, release_tag: &str) -> Result<()> { Ok(()) } -fn dist_server(nightly: bool) -> Result<()> { +fn dist_server() -> Result<()> { if cfg!(target_os = "linux") { - std::env::set_var("CC", "clang"); + env::set_var("CC", "clang"); run!( "cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --release" // We'd want to add, but that requires setting the right linker somehow // --features=jemalloc )?; - if !nightly { - run!("strip ./target/release/rust-analyzer")?; - } } else { run!("cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --release")?; } @@ -71,8 +74,20 @@ fn dist_server(nightly: bool) -> Result<()> { panic!("Unsupported OS") }; - fs2::copy(src, dst)?; + let src = Path::new(src); + let dst = Path::new(dst); + fs2::copy(&src, &dst)?; + gzip(&src, &dst.with_extension("gz"))?; + + Ok(()) +} + +fn gzip(src_path: &Path, dest_path: &Path) -> Result<()> { + let mut encoder = GzEncoder::new(File::create(dest_path)?, Compression::best()); + let mut input = io::BufReader::new(File::open(src_path)?); + io::copy(&mut input, &mut encoder)?; + encoder.finish()?; Ok(()) }