replase sparse-checkout by github api

xtask/Cargo.toml

@@ -18,3 +18,8 @@ quote = "1.0.2"
 ungrammar = "1.1.1"
 walkdir = "2.3.1"
 write-json =  "0.1.0"
+reqwest = { version = "0.10.7", features = ["blocking", "json"] }
+graphql_client = "0.9.0"
+serde = "1.0.115"
+regex = "1.3.9"
+chrono = "0.4.13"

xtask/src/codegen.rs

@@ -40,8 +40,10 @@ const AST_TOKENS: &str = "crates/ra_syntax/src/ast/generated/tokens.rs";
 const ASSISTS_DIR: &str = "crates/ra_assists/src/handlers";
 const ASSISTS_TESTS: &str = "crates/ra_assists/src/tests/generated.rs";
 
-const REPOSITORY_URL: &str = "https://github.com/rust-lang/rust";
-const UNSTABLE_FEATURE: &str = "crates/ra_ide/src/completion/unstable_feature_descriptor.rs";
+const REPO_OWNER: &str = "rust-lang";
+const REPO_NAME: &str = "rust";
+const REPO_PATH: &str = "src/doc/unstable-book/src";
+const GENERATION_DESTINATION: &str = "crates/ra_ide/src/completion/unstable_feature_descriptor.rs";
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum Mode {

xtask/src/codegen/gen_unstable_future_descriptor.rs

@@ -2,27 +2,81 @@
 
 use crate::codegen::update;
 use crate::codegen::{self, project_root, Mode, Result};
+use chrono::prelude::*;
+use fs::read_to_string;
+use graphql_client::*;
+use proc_macro2::TokenStream;
 use quote::quote;
+use regex::Regex;
+use reqwest;
+use serde::*;
 use std::fs;
+use std::fs::File;
+use std::io::copy;
+use std::io::prelude::*;
+use std::path::PathBuf;
 use std::process::Command;
 use walkdir::WalkDir;
 
-pub fn generate_unstable_future_descriptor(mode: Mode) -> Result<()> {
-    let path = project_root().join(codegen::STORAGE);
-    fs::create_dir_all(path.clone())?;
+type URI = String;
+type DateTime = String;
 
-    Command::new("git").current_dir(path.clone()).arg("init").output()?;
-    Command::new("git")
-        .current_dir(path.clone())
-        .args(&["remote", "add", "-f", "origin", codegen::REPOSITORY_URL])
-        .output()?;
-    Command::new("git")
-        .current_dir(path.clone())
-        .args(&["sparse-checkout", "set", "/src/doc/unstable-book/src/"])
-        .output()?;
-    Command::new("git").current_dir(path.clone()).args(&["pull", "origin", "master"]).output()?;
+#[derive(GraphQLQuery)]
+#[graphql(
+    schema_path = "src/codegen/schema.graphql",
+    query_path = "src/codegen/last_commit_that_affected_path.graphql"
+)]
+struct LastCommitThatAffectedPath;
 
-    let src_dir = path.join("src/doc/unstable-book/src");
+fn deep_destructuring(
+    response_body: Response<last_commit_that_affected_path::ResponseData>,
+) -> CommitInfo {
+    let t = response_body.data.unwrap().repository.unwrap().object.unwrap().on;
+
+    use last_commit_that_affected_path::LastCommitThatAffectedPathRepositoryObjectOn::Commit;
+    let commit = match t {
+        Commit(data) => data,
+        _ => panic!("type does not match"),
+    };
+    let edges = commit.history.edges.unwrap();
+    let node = edges.first().unwrap().as_ref().unwrap().node.as_ref().unwrap();
+    CommitInfo { commit_url: node.commit_url.clone(), committed_date: node.committed_date.clone() }
+}
+
+struct CommitInfo {
+    commit_url: String,
+    committed_date: String,
+}
+
+fn last_update(
+    owner: &str,
+    name: &str,
+    path: &str,
+    auth_token: Option<&str>,
+) -> Result<CommitInfo> {
+    let query =
+        LastCommitThatAffectedPath::build_query(last_commit_that_affected_path::Variables {
+            owner: owner.to_owned(),
+            name: name.to_owned(),
+            path: path.to_owned(),
+        });
+
+    let client = reqwest::blocking::Client::new();
+    let mut headers = reqwest::header::HeaderMap::new();
+    headers.insert("User-Agent", "https://github.com/rust-analyzer/rust-analyzer".parse()?);
+    let mut request = client.post("https://api.github.com/graphql").headers(headers).json(&query);
+
+    if auth_token.is_some() {
+        request = request.bearer_auth(auth_token.unwrap());
+    }
+
+    let response = request.send()?;
+
+    let response_body: Response<last_commit_that_affected_path::ResponseData> = response.json()?;
+    Ok(deep_destructuring(response_body))
+}
+
+fn generate_descriptor(src_dir: PathBuf) -> Result<TokenStream> {
     let files = WalkDir::new(src_dir.join("language-features"))
         .into_iter()
         .chain(WalkDir::new(src_dir.join("library-features")))
@@ -53,9 +107,98 @@ pub fn generate_unstable_future_descriptor(mode: Mode) -> Result<()> {
             #(#definitions),*
         ];
     };
+    Ok(ts)
+}
 
-    let destination = project_root().join(codegen::UNSTABLE_FEATURE);
-    let contents = crate::reformat(ts.to_string())?;
+fn add_anchor(text: impl std::fmt::Display, anchor: &str) -> String {
+    let anchor_str = format!(
+        r#"//The anchor is used to check if file is up to date and represent the time 
+    //of the last commit that affected path where located data for generation
+    //ANCHOR: {}"#,
+        anchor
+    );
+    format!("{}\n\n{}\n", anchor_str, text)
+}
+
+fn is_actual(path: &PathBuf, str_datetime: &str) -> bool {
+    let re = Regex::new(r"//ANCHOR: (\S*)").unwrap();
+    let opt_str = fs::read_to_string(path);
+    if opt_str.is_err() {
+        return false;
+    }
+    let text = opt_str.unwrap();
+    let opt_datetime = re.captures(text.as_str());
+    if opt_datetime.is_none() {
+        return false;
+    }
+    let str_file_dt = opt_datetime.unwrap().get(1).unwrap().as_str();
+    let file_dt = str_file_dt.parse::<chrono::DateTime<Utc>>().unwrap();
+    let datetime = str_datetime.parse::<chrono::DateTime<Utc>>().unwrap();
+
+    file_dt == datetime
+}
+
+fn download_tar(
+    owner: &str,
+    name: &str,
+    auth_token: Option<&str>,
+    destination: &PathBuf,
+    fname: &str,
+) -> Result<()> {
+    let mut headers = reqwest::header::HeaderMap::new();
+    headers.insert("User-Agent", "https://github.com/rust-analyzer/rust-analyzer".parse()?);
+    let mut request = reqwest::blocking::Client::new()
+        .get(format!("https://api.github.com/repos/{}/{}/tarball", owner, name).as_str())
+        .headers(headers);
+
+    if auth_token.is_some() {
+        request = request.bearer_auth(auth_token.unwrap());
+    }
+
+    let response = request.send()?;
+    let download_url = response.url();
+    
+    let mut response = reqwest::blocking::Client::new()
+        .get(download_url.as_str())
+        .send()?;
+
+    let mut n = fname.to_string();
+    n.push_str(".tar.gz");
+    let fpath = destination.join(n);
+    let mut file = File::create(fpath)?; 
+    response.copy_to(&mut file)?;
+
+    Ok(())
+}
+
+pub fn generate_unstable_future_descriptor(mode: Mode) -> Result<()> {
+    const auth_token: Option<&str> = None;
+
+    let path = project_root().join(codegen::STORAGE);
+    fs::create_dir_all(path.clone())?;
+
+    let commit_info =
+        last_update(codegen::REPO_OWNER, codegen::REPO_NAME, codegen::REPO_PATH, auth_token)?;
+
+    if is_actual(
+        &project_root().join(codegen::GENERATION_DESTINATION),
+        commit_info.committed_date.as_str(),
+    ) {
+        return Ok(());
+    } 
+
+    download_tar(codegen::REPO_OWNER, codegen::REPO_NAME, auth_token, &path, "repository")?;
+    Command::new("tar")
+        .args(&["-xvf", concat!("repository",".tar.gz"), "--wildcards", "*/src/doc/unstable-book/src", "--strip=1"])
+        .current_dir(codegen::STORAGE)
+        .output()?;
+
+    let src_dir = path.join(codegen::REPO_PATH);
+    let gen_holder = generate_descriptor(src_dir)?.to_string();
+    let gen_holder = add_anchor(gen_holder, commit_info.committed_date.as_str());
+
+    let destination = project_root().join(codegen::GENERATION_DESTINATION);
+    let contents = crate::reformat(gen_holder)?;
     update(destination.as_path(), &contents, mode)?;
 
     Ok(())

xtask/src/codegen/last_commit_that_affected_path.graphql

@@ -0,0 +1,17 @@
+query LastCommitThatAffectedPath($owner: String!, $name: String!, $path: String!) {
+    repository(owner: $owner, name: $name) {
+        object(expression: "master") {
+            __typename
+            ... on Commit {
+                history(path: $path, first:1) {
+                    edges {
+                        node {
+                            commitUrl
+                            committedDate
+                        }
+                    }
+                }
+            }
+        }
+    }
+}