Merge #4972

4972: Gzip artifacts r=Veetaha a=Veetaha

[Here is the test release](https://github.com/Veetaha/rust-analyzer/releases/tag/2020-06-21)

Change in size:  `~ 25 MB -> ~ 8 MB (gzipped)`

The time to gzip during the dist build takes a somewhat considerable amount of time tho.
Having already compiled artifacts this takes in debug mode:
```
~/dev/rust-analyzer (feat/gzip-binaries) $ time cargo xtask dist
    Finished dev [unoptimized] target(s) in 0.06s
     Running `target/debug/xtask dist`
> cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --release
    Finished release [optimized] target(s) in 0.05s
> strip ./target/release/rust-analyzer

real    0m34.331s
user    0m34.245s
sys     0m0.078s
```
In release mode this is much faster:
```
~/dev/rust-analyzer (feat/gzip-binaries) $ time cargo run -p xtask --release -- dist
    Finished release [optimized] target(s) in 0.04s
     Running `target/release/xtask dist`
> cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --release
    Finished release [optimized] target(s) in 0.06s
> strip ./target/release/rust-analyzer

real    0m2.401s
```

**[UPD]** adding a profile override for `miniz_oxide` does the thing to ensure good performrance

We might need to notify all other ra plugins' maintainers about the change in our GH releases if we merge this PR, or we could leave uncompressed files along with gzipped for a while until everyone migrates.

Co-authored-by: Veetaha <veetaha2@gmail.com>

Cargo.lock

@@ -207,6 +207,15 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "crc32fast"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.4.2"
@@ -340,6 +349,18 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
 
+[[package]]
+name = "flate2"
+version = "1.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68c90b0fc46cf89d227cc78b40e494ff81287a92dd07631e5af0d06fe3cf885e"
+dependencies = [
+ "cfg-if",
+ "crc32fast",
+ "libc",
+ "miniz_oxide",
+]
+
 [[package]]
 name = "flycheck"
 version = "0.1.0"
@@ -1993,6 +2014,7 @@ name = "xtask"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "flate2",
  "pico-args",
  "proc-macro2",
  "quote",

Cargo.toml

@@ -29,6 +29,11 @@ opt-level = 0
 [profile.release.package.xtask]
 opt-level = 0
 
+# Gzipping the artifacts is up to 10 times faster with optimizations (`cargo xtask dist`).
+# `miniz_oxide` is the direct dependency of `flate2` which does all the heavy lifting
+[profile.dev.package.miniz_oxide]
+opt-level = 3
+
 [patch.'crates-io']
 # rowan = { path = "../rowan" }
 

editors/code/src/main.ts

@@ -274,13 +274,13 @@ async function getServer(config: Config, state: PersistentState): Promise<string
     };
     if (config.package.releaseTag === null) return "rust-analyzer";
 
-    let binaryName: string | undefined = undefined;
+    let platform: string | undefined;
     if (process.arch === "x64" || process.arch === "ia32") {
-        if (process.platform === "linux") binaryName = "rust-analyzer-linux";
-        if (process.platform === "darwin") binaryName = "rust-analyzer-mac";
-        if (process.platform === "win32") binaryName = "rust-analyzer-windows.exe";
+        if (process.platform === "linux") platform = "linux";
+        if (process.platform === "darwin") platform = "mac";
+        if (process.platform === "win32") platform = "windows";
     }
-    if (binaryName === undefined) {
+    if (platform === undefined) {
         vscode.window.showErrorMessage(
             "Unfortunately we don't ship binaries for your platform yet. " +
             "You need to manually clone rust-analyzer repository and " +
@@ -291,8 +291,8 @@ async function getServer(config: Config, state: PersistentState): Promise<string
         );
         return undefined;
     }
-
-    const dest = path.join(config.globalStoragePath, binaryName);
+    const ext = platform === "windows" ? ".exe" : "";
+    const dest = path.join(config.globalStoragePath, `rust-analyzer-${platform}${ext}`);
     const exists = await fs.stat(dest).then(() => true, () => false);
     if (!exists) {
         await state.updateServerVersion(undefined);
@@ -309,7 +309,7 @@ async function getServer(config: Config, state: PersistentState): Promise<string
     }
 
     const release = await fetchRelease(config.package.releaseTag);
-    const artifact = release.assets.find(artifact => artifact.name === binaryName);
+    const artifact = release.assets.find(artifact => artifact.name === `rust-analyzer-${platform}.gz`);
     assert(!!artifact, `Bad release: ${JSON.stringify(release)}`);
 
     // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error.
@@ -321,6 +321,7 @@ async function getServer(config: Config, state: PersistentState): Promise<string
         url: artifact.browser_download_url,
         dest,
         progressTitle: "Downloading rust-analyzer server",
+        gunzip: true,
         mode: 0o755
     });
 

editors/code/src/net.ts

@@ -3,6 +3,7 @@ import * as vscode from "vscode";
 import * as stream from "stream";
 import * as crypto from "crypto";
 import * as fs from "fs";
+import * as zlib from "zlib";
 import * as util from "util";
 import * as path from "path";
 import { log, assert } from "./util";
@@ -65,6 +66,7 @@ interface DownloadOpts {
     url: string;
     dest: string;
     mode?: number;
+    gunzip?: boolean;
 }
 
 export async function download(opts: DownloadOpts) {
@@ -82,7 +84,7 @@ export async function download(opts: DownloadOpts) {
         },
         async (progress, _cancellationToken) => {
             let lastPercentage = 0;
-            await downloadFile(opts.url, tempFile, opts.mode, (readBytes, totalBytes) => {
+            await downloadFile(opts.url, tempFile, opts.mode, !!opts.gunzip, (readBytes, totalBytes) => {
                 const newPercentage = (readBytes / totalBytes) * 100;
                 progress.report({
                     message: newPercentage.toFixed(0) + "%",
@@ -97,16 +99,11 @@ export async function download(opts: DownloadOpts) {
     await fs.promises.rename(tempFile, opts.dest);
 }
 
-/**
- * Downloads file from `url` and stores it at `destFilePath` with `mode` (unix permissions).
- * `onProgress` callback is called on recieveing each chunk of bytes
- * to track the progress of downloading, it gets the already read and total
- * amount of bytes to read as its parameters.
- */
 async function downloadFile(
     url: string,
     destFilePath: fs.PathLike,
     mode: number | undefined,
+    gunzip: boolean,
     onProgress: (readBytes: number, totalBytes: number) => void
 ): Promise<void> {
     const res = await fetch(url);
@@ -130,7 +127,10 @@ async function downloadFile(
     });
 
     const destFileStream = fs.createWriteStream(destFilePath, { mode });
-    await pipeline(res.body, destFileStream);
+    const srcStream = gunzip ? res.body.pipe(zlib.createGunzip()) : res.body;
+
+    await pipeline(srcStream, destFileStream);
+
     await new Promise<void>(resolve => {
         destFileStream.on("close", resolve);
         destFileStream.destroy();

xtask/Cargo.toml

@@ -14,3 +14,4 @@ pico-args = "0.3.1"
 quote = "1.0.2"
 proc-macro2 = "1.0.8"
 anyhow = "1.0.26"
+flate2 = "1.0"

xtask/src/dist.rs

@@ -1,4 +1,10 @@
-use std::path::PathBuf;
+use flate2::{write::GzEncoder, Compression};
+use std::{
+    env,
+    fs::File,
+    io,
+    path::{Path, PathBuf},
+};
 
 use anyhow::Result;
 
@@ -16,7 +22,7 @@ pub fn run_dist(nightly: bool, client_version: Option<String>) -> Result<()> {
         let release_tag = if nightly { "nightly".to_string() } else { date_iso()? };
         dist_client(&version, &release_tag)?;
     }
-    dist_server(nightly)?;
+    dist_server()?;
     Ok(())
 }
 
@@ -46,17 +52,14 @@ fn dist_client(version: &str, release_tag: &str) -> Result<()> {
     Ok(())
 }
 
-fn dist_server(nightly: bool) -> Result<()> {
+fn dist_server() -> Result<()> {
     if cfg!(target_os = "linux") {
-        std::env::set_var("CC", "clang");
+        env::set_var("CC", "clang");
         run!(
             "cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --release"
             // We'd want to add, but that requires setting the right linker somehow
             // --features=jemalloc
         )?;
-        if !nightly {
-            run!("strip ./target/release/rust-analyzer")?;
-        }
     } else {
         run!("cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --release")?;
     }
@@ -71,8 +74,20 @@ fn dist_server(nightly: bool) -> Result<()> {
         panic!("Unsupported OS")
     };
 
-    fs2::copy(src, dst)?;
+    let src = Path::new(src);
+    let dst = Path::new(dst);
 
+    fs2::copy(&src, &dst)?;
+    gzip(&src, &dst.with_extension("gz"))?;
+
+    Ok(())
+}
+
+fn gzip(src_path: &Path, dest_path: &Path) -> Result<()> {
+    let mut encoder = GzEncoder::new(File::create(dest_path)?, Compression::best());
+    let mut input = io::BufReader::new(File::open(src_path)?);
+    io::copy(&mut input, &mut encoder)?;
+    encoder.finish()?;
     Ok(())
 }