Gzip artifacts

Co-authored-by: bjorn3 <bjorn3@users.noreply.github.com>

Override miniz_oxide to build it with optimizations

Building this crate with optimizations decreases the gzipping
part of `cargo xtask dist` from `30-40s` down to `3s`,
the overhead for `rustc` to apply optimizations is miserable on this background

Cargo.lock

@@ -207,6 +207,15 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "crc32fast"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.4.2"
@@ -340,6 +349,18 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
 
+[[package]]
+name = "flate2"
+version = "1.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68c90b0fc46cf89d227cc78b40e494ff81287a92dd07631e5af0d06fe3cf885e"
+dependencies = [
+ "cfg-if",
+ "crc32fast",
+ "libc",
+ "miniz_oxide",
+]
+
 [[package]]
 name = "flycheck"
 version = "0.1.0"
@@ -1993,6 +2014,7 @@ name = "xtask"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "flate2",
  "pico-args",
  "proc-macro2",
  "quote",

Cargo.toml

@@ -29,6 +29,11 @@ opt-level = 0
 [profile.release.package.xtask]
 opt-level = 0
 
+# Gzipping the artifacts is up to 10 times faster with optimizations (`cargo xtask dist`).
+# `miniz_oxide` is the direct dependency of `flate2` which does all the heavy lifting
+[profile.dev.package.miniz_oxide]
+opt-level = 3
+
 [patch.'crates-io']
 # rowan = { path = "../rowan" }
 

editors/code/src/main.ts

@@ -274,13 +274,13 @@ async function getServer(config: Config, state: PersistentState): Promise<string
     };
     if (config.package.releaseTag === null) return "rust-analyzer";
 
-    let binaryName: string | undefined = undefined;
+    let platform: string | undefined;
     if (process.arch === "x64" || process.arch === "ia32") {
-        if (process.platform === "linux") binaryName = "rust-analyzer-linux";
+        if (process.platform === "linux") platform = "linux";
-        if (process.platform === "darwin") binaryName = "rust-analyzer-mac";
+        if (process.platform === "darwin") platform = "mac";
-        if (process.platform === "win32") binaryName = "rust-analyzer-windows.exe";
+        if (process.platform === "win32") platform = "windows";
     }
-    if (binaryName === undefined) {
+    if (platform === undefined) {
         vscode.window.showErrorMessage(
             "Unfortunately we don't ship binaries for your platform yet. " +
             "You need to manually clone rust-analyzer repository and " +
@@ -291,8 +291,8 @@ async function getServer(config: Config, state: PersistentState): Promise<string
         );
         return undefined;
     }
-
+    const ext = platform === "windows" ? ".exe" : "";
-    const dest = path.join(config.globalStoragePath, binaryName);
+    const dest = path.join(config.globalStoragePath, `rust-analyzer-${platform}${ext}`);
     const exists = await fs.stat(dest).then(() => true, () => false);
     if (!exists) {
         await state.updateServerVersion(undefined);
@@ -309,7 +309,7 @@ async function getServer(config: Config, state: PersistentState): Promise<string
     }
 
     const release = await fetchRelease(config.package.releaseTag);
-    const artifact = release.assets.find(artifact => artifact.name === binaryName);
+    const artifact = release.assets.find(artifact => artifact.name === `rust-analyzer-${platform}.gz`);
     assert(!!artifact, `Bad release: ${JSON.stringify(release)}`);
 
     // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error.
@@ -321,6 +321,7 @@ async function getServer(config: Config, state: PersistentState): Promise<string
         url: artifact.browser_download_url,
         dest,
         progressTitle: "Downloading rust-analyzer server",
+        gunzip: true,
         mode: 0o755
     });
 

editors/code/src/net.ts

@@ -3,6 +3,7 @@ import * as vscode from "vscode";
 import * as stream from "stream";
 import * as crypto from "crypto";
 import * as fs from "fs";
+import * as zlib from "zlib";
 import * as util from "util";
 import * as path from "path";
 import { log, assert } from "./util";
@@ -65,6 +66,7 @@ interface DownloadOpts {
     url: string;
     dest: string;
     mode?: number;
+    gunzip?: boolean;
 }
 
 export async function download(opts: DownloadOpts) {
@@ -82,7 +84,7 @@ export async function download(opts: DownloadOpts) {
         },
         async (progress, _cancellationToken) => {
             let lastPercentage = 0;
-            await downloadFile(opts.url, tempFile, opts.mode, (readBytes, totalBytes) => {
+            await downloadFile(opts.url, tempFile, opts.mode, !!opts.gunzip, (readBytes, totalBytes) => {
                 const newPercentage = (readBytes / totalBytes) * 100;
                 progress.report({
                     message: newPercentage.toFixed(0) + "%",
@@ -97,16 +99,11 @@ export async function download(opts: DownloadOpts) {
     await fs.promises.rename(tempFile, opts.dest);
 }
 
-/**
- * Downloads file from `url` and stores it at `destFilePath` with `mode` (unix permissions).
- * `onProgress` callback is called on recieveing each chunk of bytes
- * to track the progress of downloading, it gets the already read and total
- * amount of bytes to read as its parameters.
- */
 async function downloadFile(
     url: string,
     destFilePath: fs.PathLike,
     mode: number | undefined,
+    gunzip: boolean,
     onProgress: (readBytes: number, totalBytes: number) => void
 ): Promise<void> {
     const res = await fetch(url);
@@ -130,7 +127,10 @@ async function downloadFile(
     });
 
     const destFileStream = fs.createWriteStream(destFilePath, { mode });
-    await pipeline(res.body, destFileStream);
+    const srcStream = gunzip ? res.body.pipe(zlib.createGunzip()) : res.body;
+
+    await pipeline(srcStream, destFileStream);
+
     await new Promise<void>(resolve => {
         destFileStream.on("close", resolve);
         destFileStream.destroy();

xtask/Cargo.toml

@@ -14,3 +14,4 @@ pico-args = "0.3.1"
 quote = "1.0.2"
 proc-macro2 = "1.0.8"
 anyhow = "1.0.26"
+flate2 = "1.0"

xtask/src/dist.rs

@@ -1,4 +1,10 @@
-use std::path::PathBuf;
+use flate2::{write::GzEncoder, Compression};
+use std::{
+    env,
+    fs::File,
+    io,
+    path::{Path, PathBuf},
+};
 
 use anyhow::Result;
 
@@ -16,7 +22,7 @@ pub fn run_dist(nightly: bool, client_version: Option<String>) -> Result<()> {
         let release_tag = if nightly { "nightly".to_string() } else { date_iso()? };
         dist_client(&version, &release_tag)?;
     }
-    dist_server(nightly)?;
+    dist_server()?;
     Ok(())
 }
 
@@ -46,17 +52,14 @@ fn dist_client(version: &str, release_tag: &str) -> Result<()> {
     Ok(())
 }
 
-fn dist_server(nightly: bool) -> Result<()> {
+fn dist_server() -> Result<()> {
     if cfg!(target_os = "linux") {
-        std::env::set_var("CC", "clang");
+        env::set_var("CC", "clang");
         run!(
             "cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --release"
             // We'd want to add, but that requires setting the right linker somehow
             // --features=jemalloc
         )?;
-        if !nightly {
-            run!("strip ./target/release/rust-analyzer")?;
-        }
     } else {
         run!("cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --release")?;
     }
@@ -71,8 +74,20 @@ fn dist_server(nightly: bool) -> Result<()> {
         panic!("Unsupported OS")
     };
 
-    fs2::copy(src, dst)?;
+    let src = Path::new(src);
+    let dst = Path::new(dst);
 
+    fs2::copy(&src, &dst)?;
+    gzip(&src, &dst.with_extension("gz"))?;
+
+    Ok(())
+}
+
+fn gzip(src_path: &Path, dest_path: &Path) -> Result<()> {
+    let mut encoder = GzEncoder::new(File::create(dest_path)?, Compression::best());
+    let mut input = io::BufReader::new(File::open(src_path)?);
+    io::copy(&mut input, &mut encoder)?;
+    encoder.finish()?;
     Ok(())
 }