From b93ced6f633fab2733b40aef2541582b00e053fb Mon Sep 17 00:00:00 2001 From: Matthias Einwag Date: Tue, 22 Sep 2020 23:12:51 -0700 Subject: [PATCH 01/11] Allow to use a Github Auth token for fetching releases This change allows to use a authorization token provided by Github in order to fetch metadata for a RA release. Using an authorization token prevents to get rate-limited in environments where lots of RA users use a shared client IP (e.g. behind a company NAT). The auth token is stored in `ExtensionContext.globalState`. As far as I could observe through testing with a local WSL2 environment that state is synced between an extension installed locally and a remote version. The change provides no explicit command to query for an auth token. However in case a download fails it will provide a retry option as well as an option to enter the auth token. This should be more discoverable for most users. Closes #3688 --- editors/code/src/main.ts | 55 +++++++++++++++++++++++++++- editors/code/src/net.ts | 10 ++++- editors/code/src/persistent_state.ts | 11 ++++++ 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts index bd99d696ad..8c16105706 100644 --- a/editors/code/src/main.ts +++ b/editors/code/src/main.ts @@ -173,7 +173,9 @@ async function bootstrapExtension(config: Config, state: PersistentState): Promi if (!shouldCheckForNewNightly) return; } - const release = await fetchRelease("nightly").catch((e) => { + const release = await performDownloadWithRetryDialog(async () => { + return await fetchRelease("nightly", state.githubToken); + }, state).catch((e) => { log.error(e); if (state.releaseId === undefined) { // Show error only for the initial download vscode.window.showErrorMessage(`Failed to download rust-analyzer nightly ${e}`); @@ -308,7 +310,10 @@ async function getServer(config: Config, state: PersistentState): Promise { + return await fetchRelease(releaseTag, state.githubToken); + }, state); const artifact = release.assets.find(artifact => artifact.name === `rust-analyzer-${platform}.gz`); assert(!!artifact, `Bad release: ${JSON.stringify(release)}`); @@ -333,3 +338,49 @@ async function getServer(config: Config, state: PersistentState): Promise(downloadFunc: () => Promise, state: PersistentState): Promise { + while (true) { + try { + return await downloadFunc(); + } catch (e) { + let selected = await vscode.window.showErrorMessage("Failed perform download: " + e.message, {}, { + title: "Update Github Auth Token", + updateToken: true, + }, { + title: "Retry download", + retry: true, + }, { + title: "Dismiss", + }); + + if (selected?.updateToken) { + await queryForGithubToken(state); + continue; + } else if (selected?.retry) { + continue; + } + throw e; + }; + } + +} + +async function queryForGithubToken(state: PersistentState): Promise { + const githubTokenOptions: vscode.InputBoxOptions = { + value: state.githubToken, + password: true, + prompt: ` + This dialog allows to store a Github authorization token. + The usage of an authorization token allows will increase the rate + limit on the use of Github APIs and can thereby prevent getting + throttled. + Auth tokens can be obtained at https://github.com/settings/tokens`, + }; + + const newToken = await vscode.window.showInputBox(githubTokenOptions); + if (newToken) { + log.info("Storing new github token"); + await state.updateGithubToken(newToken); + } +} \ No newline at end of file diff --git a/editors/code/src/net.ts b/editors/code/src/net.ts index 5eba2728d2..d6194b63e4 100644 --- a/editors/code/src/net.ts +++ b/editors/code/src/net.ts @@ -18,7 +18,8 @@ const OWNER = "rust-analyzer"; const REPO = "rust-analyzer"; export async function fetchRelease( - releaseTag: string + releaseTag: string, + githubToken: string | null | undefined, ): Promise { const apiEndpointPath = `/repos/${OWNER}/${REPO}/releases/tags/${releaseTag}`; @@ -27,7 +28,12 @@ export async function fetchRelease( log.debug("Issuing request for released artifacts metadata to", requestUrl); - const response = await fetch(requestUrl, { headers: { Accept: "application/vnd.github.v3+json" } }); + var headers: any = { Accept: "application/vnd.github.v3+json" }; + if (githubToken != null) { + headers.Authorization = "token " + githubToken; + } + + const response = await fetch(requestUrl, { headers: headers }); if (!response.ok) { log.error("Error fetching artifact release info", { diff --git a/editors/code/src/persistent_state.ts b/editors/code/src/persistent_state.ts index 5705eed812..afb6525899 100644 --- a/editors/code/src/persistent_state.ts +++ b/editors/code/src/persistent_state.ts @@ -38,4 +38,15 @@ export class PersistentState { async updateServerVersion(value: string | undefined) { await this.globalState.update("serverVersion", value); } + + /** + * Github authorization token. + * This is used for API requests against the Github API. + */ + get githubToken(): string | undefined { + return this.globalState.get("githubToken"); + } + async updateGithubToken(value: string | undefined) { + await this.globalState.update("githubToken", value); + } } From 1503d9de411347752fab7313e53d2061fa0186b1 Mon Sep 17 00:00:00 2001 From: Matthias Einwag Date: Tue, 22 Sep 2020 23:41:51 -0700 Subject: [PATCH 02/11] Fix tslint --- editors/code/src/main.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts index 8c16105706..9743115fba 100644 --- a/editors/code/src/main.ts +++ b/editors/code/src/main.ts @@ -344,7 +344,7 @@ async function performDownloadWithRetryDialog(downloadFunc: () => Promise, try { return await downloadFunc(); } catch (e) { - let selected = await vscode.window.showErrorMessage("Failed perform download: " + e.message, {}, { + const selected = await vscode.window.showErrorMessage("Failed perform download: " + e.message, {}, { title: "Update Github Auth Token", updateToken: true, }, { @@ -353,7 +353,7 @@ async function performDownloadWithRetryDialog(downloadFunc: () => Promise, }, { title: "Dismiss", }); - + if (selected?.updateToken) { await queryForGithubToken(state); continue; From a0a7cd306ef6d9476b37b85365418f84c374ae59 Mon Sep 17 00:00:00 2001 From: Matthias Einwag Date: Wed, 23 Sep 2020 00:28:38 -0700 Subject: [PATCH 03/11] Use retry dialog also for downloads Since the change already implements a retry dialog for network operations, let's also use it for allowing to retry the actual file. --- editors/code/src/main.ts | 48 ++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts index 9743115fba..409e4b5c26 100644 --- a/editors/code/src/main.ts +++ b/editors/code/src/main.ts @@ -194,11 +194,19 @@ async function bootstrapExtension(config: Config, state: PersistentState): Promi assert(!!artifact, `Bad release: ${JSON.stringify(release)}`); const dest = path.join(config.globalStoragePath, "rust-analyzer.vsix"); - await download({ - url: artifact.browser_download_url, - dest, - progressTitle: "Downloading rust-analyzer extension", - }); + + await performDownloadWithRetryDialog(async () => { + // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error. + await fs.unlink(dest).catch(err => { + if (err.code !== "ENOENT") throw err; + }); + + await download({ + url: artifact.browser_download_url, + dest, + progressTitle: "Downloading rust-analyzer extension", + }); + }, state); await vscode.commands.executeCommand("workbench.extensions.installExtension", vscode.Uri.file(dest)); await fs.unlink(dest); @@ -317,18 +325,20 @@ async function getServer(config: Config, state: PersistentState): Promise artifact.name === `rust-analyzer-${platform}.gz`); assert(!!artifact, `Bad release: ${JSON.stringify(release)}`); - // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error. - await fs.unlink(dest).catch(err => { - if (err.code !== "ENOENT") throw err; - }); + await performDownloadWithRetryDialog(async () => { + // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error. + await fs.unlink(dest).catch(err => { + if (err.code !== "ENOENT") throw err; + }); - await download({ - url: artifact.browser_download_url, - dest, - progressTitle: "Downloading rust-analyzer server", - gunzip: true, - mode: 0o755 - }); + await download({ + url: artifact.browser_download_url, + dest, + progressTitle: "Downloading rust-analyzer server", + gunzip: true, + mode: 0o755 + }); + }, state); // Patching executable if that's NixOS. if (await fs.stat("/etc/nixos").then(_ => true).catch(_ => false)) { @@ -372,10 +382,10 @@ async function queryForGithubToken(state: PersistentState): Promise { password: true, prompt: ` This dialog allows to store a Github authorization token. - The usage of an authorization token allows will increase the rate + The usage of an authorization token will increase the rate limit on the use of Github APIs and can thereby prevent getting throttled. - Auth tokens can be obtained at https://github.com/settings/tokens`, + Auth tokens can be created at https://github.com/settings/tokens`, }; const newToken = await vscode.window.showInputBox(githubTokenOptions); @@ -383,4 +393,4 @@ async function queryForGithubToken(state: PersistentState): Promise { log.info("Storing new github token"); await state.updateGithubToken(newToken); } -} \ No newline at end of file +} From 501b516db4a9a50c39e2fb90b389d77c9541e43f Mon Sep 17 00:00:00 2001 From: Matthias Einwag Date: Wed, 23 Sep 2020 00:50:34 -0700 Subject: [PATCH 04/11] Add a command for updating the Github API token --- editors/code/package.json | 9 +++++++++ editors/code/src/main.ts | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/editors/code/package.json b/editors/code/package.json index c57fbdda2a..1326649266 100644 --- a/editors/code/package.json +++ b/editors/code/package.json @@ -158,6 +158,11 @@ "title": "Restart server", "category": "Rust Analyzer" }, + { + "command": "rust-analyzer.updateGithubToken", + "title": "Update Github API token", + "category": "Rust Analyzer" + }, { "command": "rust-analyzer.onEnter", "title": "Enhanced enter key", @@ -984,6 +989,10 @@ "command": "rust-analyzer.reload", "when": "inRustProject" }, + { + "command": "rust-analyzer.updateGithubToken", + "when": "inRustProject" + }, { "command": "rust-analyzer.onEnter", "when": "inRustProject" diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts index 409e4b5c26..2fcd853d44 100644 --- a/editors/code/src/main.ts +++ b/editors/code/src/main.ts @@ -95,6 +95,10 @@ async function tryActivate(context: vscode.ExtensionContext) { await activate(context).catch(log.error); }); + ctx.registerCommand('updateGithubToken', ctx => async () => { + await queryForGithubToken(new PersistentState(ctx.globalState)); + }); + ctx.registerCommand('analyzerStatus', commands.analyzerStatus); ctx.registerCommand('memoryUsage', commands.memoryUsage); ctx.registerCommand('reloadWorkspace', commands.reloadWorkspace); From 145bd6f70138246b4e5efebcd94786f147ac9e7a Mon Sep 17 00:00:00 2001 From: Matthias Einwag Date: Wed, 23 Sep 2020 01:03:34 -0700 Subject: [PATCH 05/11] Fix clearing the token The previous version would have interpreted an empty token as an abort of the dialog and would have not properly cleared the token. This is now fixed by checking for `undefined` for a an abort and by setting the token to `undefined` in order to clear it. --- editors/code/src/main.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts index 2fcd853d44..ce7c56d05a 100644 --- a/editors/code/src/main.ts +++ b/editors/code/src/main.ts @@ -393,8 +393,13 @@ async function queryForGithubToken(state: PersistentState): Promise { }; const newToken = await vscode.window.showInputBox(githubTokenOptions); - if (newToken) { - log.info("Storing new github token"); - await state.updateGithubToken(newToken); + if (newToken !== undefined) { + if (newToken === "") { + log.info("Clearing github token"); + await state.updateGithubToken(undefined); + } else { + log.info("Storing new github token"); + await state.updateGithubToken(newToken); + } } } From 45de3e738c33e972540643f470d6163052219d84 Mon Sep 17 00:00:00 2001 From: Matthias Einwag Date: Wed, 23 Sep 2020 01:06:10 -0700 Subject: [PATCH 06/11] Remove stray newline --- editors/code/src/main.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts index ce7c56d05a..5a33b8fc29 100644 --- a/editors/code/src/main.ts +++ b/editors/code/src/main.ts @@ -377,7 +377,6 @@ async function performDownloadWithRetryDialog(downloadFunc: () => Promise, throw e; }; } - } async function queryForGithubToken(state: PersistentState): Promise { From 87933e15ce3b7a603b6e28597cdc152669e90cca Mon Sep 17 00:00:00 2001 From: Matthias Einwag Date: Wed, 23 Sep 2020 08:14:18 -0700 Subject: [PATCH 07/11] Apply suggestions from code review Co-authored-by: Veetaha --- editors/code/src/main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts index 5a33b8fc29..72c545b3c1 100644 --- a/editors/code/src/main.ts +++ b/editors/code/src/main.ts @@ -358,7 +358,7 @@ async function performDownloadWithRetryDialog(downloadFunc: () => Promise, try { return await downloadFunc(); } catch (e) { - const selected = await vscode.window.showErrorMessage("Failed perform download: " + e.message, {}, { + const selected = await vscode.window.showErrorMessage("Failed to download: " + e.message, {}, { title: "Update Github Auth Token", updateToken: true, }, { From d38f759c631039d11cb490692b5e07b00324ff10 Mon Sep 17 00:00:00 2001 From: Matthias Einwag Date: Wed, 23 Sep 2020 08:24:35 -0700 Subject: [PATCH 08/11] Use closure in trailing position and strongly type header map --- editors/code/src/main.ts | 37 ++++++++++++++++++++----------------- editors/code/src/net.ts | 2 +- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts index 72c545b3c1..0ee5280cc0 100644 --- a/editors/code/src/main.ts +++ b/editors/code/src/main.ts @@ -177,9 +177,9 @@ async function bootstrapExtension(config: Config, state: PersistentState): Promi if (!shouldCheckForNewNightly) return; } - const release = await performDownloadWithRetryDialog(async () => { + const release = await performDownloadWithRetryDialog(state, async () => { return await fetchRelease("nightly", state.githubToken); - }, state).catch((e) => { + }).catch((e) => { log.error(e); if (state.releaseId === undefined) { // Show error only for the initial download vscode.window.showErrorMessage(`Failed to download rust-analyzer nightly ${e}`); @@ -199,7 +199,7 @@ async function bootstrapExtension(config: Config, state: PersistentState): Promi const dest = path.join(config.globalStoragePath, "rust-analyzer.vsix"); - await performDownloadWithRetryDialog(async () => { + await performDownloadWithRetryDialog(state, async () => { // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error. await fs.unlink(dest).catch(err => { if (err.code !== "ENOENT") throw err; @@ -210,7 +210,7 @@ async function bootstrapExtension(config: Config, state: PersistentState): Promi dest, progressTitle: "Downloading rust-analyzer extension", }); - }, state); + }); await vscode.commands.executeCommand("workbench.extensions.installExtension", vscode.Uri.file(dest)); await fs.unlink(dest); @@ -323,13 +323,13 @@ async function getServer(config: Config, state: PersistentState): Promise { + const release = await performDownloadWithRetryDialog(state, async () => { return await fetchRelease(releaseTag, state.githubToken); - }, state); + }); const artifact = release.assets.find(artifact => artifact.name === `rust-analyzer-${platform}.gz`); assert(!!artifact, `Bad release: ${JSON.stringify(release)}`); - await performDownloadWithRetryDialog(async () => { + await performDownloadWithRetryDialog(state, async () => { // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error. await fs.unlink(dest).catch(err => { if (err.code !== "ENOENT") throw err; @@ -342,7 +342,7 @@ async function getServer(config: Config, state: PersistentState): Promise true).catch(_ => false)) { @@ -353,7 +353,7 @@ async function getServer(config: Config, state: PersistentState): Promise(downloadFunc: () => Promise, state: PersistentState): Promise { +async function performDownloadWithRetryDialog(state: PersistentState, downloadFunc: () => Promise): Promise { while (true) { try { return await downloadFunc(); @@ -392,13 +392,16 @@ async function queryForGithubToken(state: PersistentState): Promise { }; const newToken = await vscode.window.showInputBox(githubTokenOptions); - if (newToken !== undefined) { - if (newToken === "") { - log.info("Clearing github token"); - await state.updateGithubToken(undefined); - } else { - log.info("Storing new github token"); - await state.updateGithubToken(newToken); - } + if (newToken === undefined) { + // The user aborted the dialog => Do not update the stored token + return; + } + + if (newToken === "") { + log.info("Clearing github token"); + await state.updateGithubToken(undefined); + } else { + log.info("Storing new github token"); + await state.updateGithubToken(newToken); } } diff --git a/editors/code/src/net.ts b/editors/code/src/net.ts index d6194b63e4..cfbe1fd486 100644 --- a/editors/code/src/net.ts +++ b/editors/code/src/net.ts @@ -28,7 +28,7 @@ export async function fetchRelease( log.debug("Issuing request for released artifacts metadata to", requestUrl); - var headers: any = { Accept: "application/vnd.github.v3+json" }; + const headers: Record = { Accept: "application/vnd.github.v3+json" }; if (githubToken != null) { headers.Authorization = "token " + githubToken; } From df4d59512e496ff010c8710e8ea8e2db4a7f4822 Mon Sep 17 00:00:00 2001 From: Matthias Einwag Date: Wed, 23 Sep 2020 08:27:25 -0700 Subject: [PATCH 09/11] Remane function --- editors/code/src/main.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts index 0ee5280cc0..f865639a14 100644 --- a/editors/code/src/main.ts +++ b/editors/code/src/main.ts @@ -177,7 +177,7 @@ async function bootstrapExtension(config: Config, state: PersistentState): Promi if (!shouldCheckForNewNightly) return; } - const release = await performDownloadWithRetryDialog(state, async () => { + const release = await downloadWithRetryDialog(state, async () => { return await fetchRelease("nightly", state.githubToken); }).catch((e) => { log.error(e); @@ -199,7 +199,7 @@ async function bootstrapExtension(config: Config, state: PersistentState): Promi const dest = path.join(config.globalStoragePath, "rust-analyzer.vsix"); - await performDownloadWithRetryDialog(state, async () => { + await downloadWithRetryDialog(state, async () => { // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error. await fs.unlink(dest).catch(err => { if (err.code !== "ENOENT") throw err; @@ -323,13 +323,13 @@ async function getServer(config: Config, state: PersistentState): Promise { + const release = await downloadWithRetryDialog(state, async () => { return await fetchRelease(releaseTag, state.githubToken); }); const artifact = release.assets.find(artifact => artifact.name === `rust-analyzer-${platform}.gz`); assert(!!artifact, `Bad release: ${JSON.stringify(release)}`); - await performDownloadWithRetryDialog(state, async () => { + await downloadWithRetryDialog(state, async () => { // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error. await fs.unlink(dest).catch(err => { if (err.code !== "ENOENT") throw err; @@ -353,7 +353,7 @@ async function getServer(config: Config, state: PersistentState): Promise(state: PersistentState, downloadFunc: () => Promise): Promise { +async function downloadWithRetryDialog(state: PersistentState, downloadFunc: () => Promise): Promise { while (true) { try { return await downloadFunc(); From c7f464774901d40483a6edc4f1294e1648dee4d5 Mon Sep 17 00:00:00 2001 From: Matthias Einwag Date: Wed, 23 Sep 2020 08:37:02 -0700 Subject: [PATCH 10/11] Move unlink on download into download function Since this is required by all callsites its easier to have it in the function itself. --- editors/code/src/main.ts | 14 +++----------- editors/code/src/net.ts | 8 ++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts index f865639a14..2896d90ac9 100644 --- a/editors/code/src/main.ts +++ b/editors/code/src/main.ts @@ -200,15 +200,11 @@ async function bootstrapExtension(config: Config, state: PersistentState): Promi const dest = path.join(config.globalStoragePath, "rust-analyzer.vsix"); await downloadWithRetryDialog(state, async () => { - // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error. - await fs.unlink(dest).catch(err => { - if (err.code !== "ENOENT") throw err; - }); - await download({ url: artifact.browser_download_url, dest, progressTitle: "Downloading rust-analyzer extension", + overwrite: true, }); }); @@ -330,17 +326,13 @@ async function getServer(config: Config, state: PersistentState): Promise { - // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error. - await fs.unlink(dest).catch(err => { - if (err.code !== "ENOENT") throw err; - }); - await download({ url: artifact.browser_download_url, dest, progressTitle: "Downloading rust-analyzer server", gunzip: true, - mode: 0o755 + mode: 0o755, + overwrite: true, }); }); diff --git a/editors/code/src/net.ts b/editors/code/src/net.ts index cfbe1fd486..e746465d1d 100644 --- a/editors/code/src/net.ts +++ b/editors/code/src/net.ts @@ -76,6 +76,7 @@ interface DownloadOpts { dest: string; mode?: number; gunzip?: boolean; + overwrite?: boolean, } export async function download(opts: DownloadOpts) { @@ -85,6 +86,13 @@ export async function download(opts: DownloadOpts) { const randomHex = crypto.randomBytes(5).toString("hex"); const tempFile = path.join(dest.dir, `${dest.name}${randomHex}`); + if (opts.overwrite) { + // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error. + await fs.promises.unlink(opts.dest).catch(err => { + if (err.code !== "ENOENT") throw err; + }); + } + await vscode.window.withProgress( { location: vscode.ProgressLocation.Notification, From 8eae893c767941bf02338cd74d7b103437783013 Mon Sep 17 00:00:00 2001 From: Matthias Einwag Date: Wed, 23 Sep 2020 08:39:04 -0700 Subject: [PATCH 11/11] Fix lint --- editors/code/src/net.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/editors/code/src/net.ts b/editors/code/src/net.ts index e746465d1d..9ba17b7b5d 100644 --- a/editors/code/src/net.ts +++ b/editors/code/src/net.ts @@ -76,7 +76,7 @@ interface DownloadOpts { dest: string; mode?: number; gunzip?: boolean; - overwrite?: boolean, + overwrite?: boolean; } export async function download(opts: DownloadOpts) {