feat: emit SCIP via rust-analyzer

This commit is contained in:
TJ DeVries 2022-06-10 21:29:04 -04:00
parent 6711ded5cd
commit 50ecb09da4
8 changed files with 630 additions and 14 deletions

50
Cargo.lock generated
View file

@ -1229,6 +1229,26 @@ dependencies = [
"tracing",
]
[[package]]
name = "protobuf"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ee4a7d8b91800c8f167a6268d1a1026607368e1adc84e98fe044aeb905302f7"
dependencies = [
"once_cell",
"protobuf-support",
"thiserror",
]
[[package]]
name = "protobuf-support"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ca157fe12fc7ee2e315f2f735e27df41b3d97cdd70ea112824dac1ffb08ee1c"
dependencies = [
"thiserror",
]
[[package]]
name = "pulldown-cmark"
version = "0.9.2"
@ -1360,6 +1380,7 @@ dependencies = [
"project-model",
"rayon",
"rustc-hash",
"scip",
"serde",
"serde_json",
"sourcegen",
@ -1446,6 +1467,15 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "scip"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2bfbb10286f69fad7c78db71004b7839bf957788359fe0c479f029f9849136b"
dependencies = [
"protobuf",
]
[[package]]
name = "scoped-tls"
version = "1.0.0"
@ -1631,6 +1661,26 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "288cb548dbe72b652243ea797201f3d481a0609a967980fcc5b2315ea811560a"
[[package]]
name = "thiserror"
version = "1.0.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "thread_local"
version = "1.1.4"

View file

@ -87,7 +87,7 @@ pub use crate::{
},
join_lines::JoinLinesConfig,
markup::Markup,
moniker::{MonikerKind, MonikerResult, PackageInformation},
moniker::{MonikerDescriptorKind, MonikerKind, MonikerResult, PackageInformation},
move_item::Direction,
navigation_target::NavigationTarget,
prime_caches::ParallelPrimeCachesProgress,

View file

@ -13,17 +13,39 @@ use syntax::{AstNode, SyntaxKind::*, T};
use crate::{doc_links::token_as_doc_comment, RangeInfo};
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum MonikerDescriptorKind {
Namespace,
Type,
Term,
Method,
TypeParameter,
Parameter,
Macro,
Meta,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct MonikerDescriptor {
pub name: Name,
pub desc: MonikerDescriptorKind,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct MonikerIdentifier {
crate_name: String,
path: Vec<Name>,
pub crate_name: String,
pub description: Vec<MonikerDescriptor>,
}
impl ToString for MonikerIdentifier {
fn to_string(&self) -> String {
match self {
MonikerIdentifier { path, crate_name } => {
format!("{}::{}", crate_name, path.iter().map(|x| x.to_string()).join("::"))
MonikerIdentifier { description, crate_name } => {
format!(
"{}::{}",
crate_name,
description.iter().map(|x| x.name.to_string()).join("::")
)
}
}
}
@ -42,6 +64,12 @@ pub struct MonikerResult {
pub package_information: PackageInformation,
}
impl MonikerResult {
pub fn from_def(db: &RootDatabase, def: Definition, from_crate: Crate) -> Option<Self> {
def_to_moniker(db, def, from_crate)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct PackageInformation {
pub name: String,
@ -105,13 +133,23 @@ pub(crate) fn def_to_moniker(
def: Definition,
from_crate: Crate,
) -> Option<MonikerResult> {
if matches!(def, Definition::GenericParam(_) | Definition::SelfType(_) | Definition::Local(_)) {
if matches!(
def,
Definition::GenericParam(_)
| Definition::Label(_)
| Definition::DeriveHelper(_)
| Definition::BuiltinAttr(_)
| Definition::ToolModule(_)
) {
return None;
}
let module = def.module(db)?;
let krate = module.krate();
let mut path = vec![];
path.extend(module.path_to_root(db).into_iter().filter_map(|x| x.name(db)));
let mut description = vec![];
description.extend(module.path_to_root(db).into_iter().filter_map(|x| {
Some(MonikerDescriptor { name: x.name(db)?, desc: MonikerDescriptorKind::Namespace })
}));
// Handle associated items within a trait
if let Some(assoc) = def.as_assoc_item(db) {
@ -120,31 +158,98 @@ pub(crate) fn def_to_moniker(
AssocItemContainer::Trait(trait_) => {
// Because different traits can have functions with the same name,
// we have to include the trait name as part of the moniker for uniqueness.
path.push(trait_.name(db));
description.push(MonikerDescriptor {
name: trait_.name(db),
desc: MonikerDescriptorKind::Type,
});
}
AssocItemContainer::Impl(impl_) => {
// Because a struct can implement multiple traits, for implementations
// we add both the struct name and the trait name to the path
if let Some(adt) = impl_.self_ty(db).as_adt() {
path.push(adt.name(db));
description.push(MonikerDescriptor {
name: adt.name(db),
desc: MonikerDescriptorKind::Type,
});
}
if let Some(trait_) = impl_.trait_(db) {
path.push(trait_.name(db));
description.push(MonikerDescriptor {
name: trait_.name(db),
desc: MonikerDescriptorKind::Type,
});
}
}
}
}
if let Definition::Field(it) = def {
path.push(it.parent_def(db).name(db));
description.push(MonikerDescriptor {
name: it.parent_def(db).name(db),
desc: MonikerDescriptorKind::Type,
});
}
path.push(def.name(db)?);
let name_desc = match def {
// These are handled by top-level guard (for performance).
Definition::GenericParam(_)
| Definition::Label(_)
| Definition::DeriveHelper(_)
| Definition::BuiltinAttr(_)
| Definition::ToolModule(_) => return None,
Definition::Local(local) => {
if !local.is_param(db) {
return None;
}
MonikerDescriptor { name: local.name(db), desc: MonikerDescriptorKind::Parameter }
}
Definition::Macro(m) => {
MonikerDescriptor { name: m.name(db), desc: MonikerDescriptorKind::Macro }
}
Definition::Function(f) => {
MonikerDescriptor { name: f.name(db), desc: MonikerDescriptorKind::Method }
}
Definition::Variant(v) => {
MonikerDescriptor { name: v.name(db), desc: MonikerDescriptorKind::Type }
}
Definition::Const(c) => {
MonikerDescriptor { name: c.name(db)?, desc: MonikerDescriptorKind::Term }
}
Definition::Trait(trait_) => {
MonikerDescriptor { name: trait_.name(db), desc: MonikerDescriptorKind::Type }
}
Definition::TypeAlias(ta) => {
MonikerDescriptor { name: ta.name(db), desc: MonikerDescriptorKind::TypeParameter }
}
Definition::Module(m) => {
MonikerDescriptor { name: m.name(db)?, desc: MonikerDescriptorKind::Namespace }
}
Definition::BuiltinType(b) => {
MonikerDescriptor { name: b.name(), desc: MonikerDescriptorKind::Type }
}
Definition::SelfType(imp) => MonikerDescriptor {
name: imp.self_ty(db).as_adt()?.name(db),
desc: MonikerDescriptorKind::Type,
},
Definition::Field(it) => {
MonikerDescriptor { name: it.name(db), desc: MonikerDescriptorKind::Term }
}
Definition::Adt(adt) => {
MonikerDescriptor { name: adt.name(db), desc: MonikerDescriptorKind::Type }
}
Definition::Static(s) => {
MonikerDescriptor { name: s.name(db), desc: MonikerDescriptorKind::Meta }
}
};
description.push(name_desc);
Some(MonikerResult {
identifier: MonikerIdentifier {
crate_name: krate.display_name(db)?.crate_name().to_string(),
path,
description,
},
kind: if krate == from_crate { MonikerKind::Export } else { MonikerKind::Import },
package_information: {

View file

@ -23,6 +23,7 @@ crossbeam-channel = "0.5.5"
dissimilar = "1.0.4"
itertools = "0.10.3"
lsp-types = { version = "0.93.0", features = ["proposed"] }
scip = "0.1.1"
parking_lot = "0.12.1"
xflags = "0.2.4"
oorandom = "11.1.3"

View file

@ -93,6 +93,7 @@ fn try_main() -> Result<()> {
flags::RustAnalyzerCmd::Ssr(cmd) => cmd.run()?,
flags::RustAnalyzerCmd::Search(cmd) => cmd.run()?,
flags::RustAnalyzerCmd::Lsif(cmd) => cmd.run()?,
flags::RustAnalyzerCmd::Scip(cmd) => cmd.run()?,
}
Ok(())
}

View file

@ -9,6 +9,7 @@ mod analysis_stats;
mod diagnostics;
mod ssr;
mod lsif;
mod scip;
mod progress_report;

View file

@ -112,6 +112,10 @@ xflags::xflags! {
cmd lsif
required path: PathBuf
{}
cmd scip
required path: PathBuf
{}
}
}
@ -140,6 +144,7 @@ pub enum RustAnalyzerCmd {
Search(Search),
ProcMacro(ProcMacro),
Lsif(Lsif),
Scip(Scip),
}
#[derive(Debug)]
@ -207,6 +212,11 @@ pub struct Lsif {
pub path: PathBuf,
}
#[derive(Debug)]
pub struct Scip {
pub path: PathBuf,
}
impl RustAnalyzer {
pub const HELP: &'static str = Self::HELP_;

View file

@ -0,0 +1,448 @@
//! SCIP generator
use std::{
collections::{HashMap, HashSet},
time::Instant,
};
use crate::line_index::{LineEndings, LineIndex, OffsetEncoding};
use hir::Name;
use ide::{
LineCol, MonikerDescriptorKind, MonikerResult, StaticIndex, StaticIndexedFile, TextRange,
TokenId,
};
use ide_db::LineIndexDatabase;
use project_model::{CargoConfig, ProjectManifest, ProjectWorkspace};
use scip::types as scip_types;
use std::env;
use crate::cli::{
flags,
load_cargo::{load_workspace, LoadCargoConfig},
Result,
};
impl flags::Scip {
pub fn run(self) -> Result<()> {
eprintln!("Generating SCIP start...");
let now = Instant::now();
let cargo_config = CargoConfig::default();
let no_progress = &|s| (eprintln!("rust-analyzer: Loading {}", s));
let load_cargo_config = LoadCargoConfig {
load_out_dirs_from_check: true,
with_proc_macro: true,
prefill_caches: true,
};
let path = vfs::AbsPathBuf::assert(env::current_dir()?.join(&self.path));
let rootpath = path.normalize();
let manifest = ProjectManifest::discover_single(&path)?;
let workspace = ProjectWorkspace::load(manifest, &cargo_config, no_progress)?;
let (host, vfs, _) = load_workspace(workspace, &load_cargo_config)?;
let db = host.raw_database();
let analysis = host.analysis();
let si = StaticIndex::compute(&analysis);
let mut index = scip_types::Index {
metadata: Some(scip_types::Metadata {
version: scip_types::ProtocolVersion::UnspecifiedProtocolVersion.into(),
tool_info: Some(scip_types::ToolInfo {
name: "rust-analyzer".to_owned(),
version: "0.1".to_owned(),
arguments: vec![],
..Default::default()
})
.into(),
project_root: format!(
"file://{}",
path.normalize()
.as_os_str()
.to_str()
.ok_or(anyhow::anyhow!("Unable to normalize project_root path"))?
.to_string()
),
text_document_encoding: scip_types::TextEncoding::UTF8.into(),
..Default::default()
})
.into(),
..Default::default()
};
let mut symbols_emitted: HashSet<TokenId> = HashSet::default();
let mut tokens_to_symbol: HashMap<TokenId, String> = HashMap::new();
for file in si.files {
let mut local_count = 0;
let mut new_local_symbol = || {
let new_symbol = scip::types::Symbol::new_local(local_count);
local_count += 1;
new_symbol
};
let StaticIndexedFile { file_id, tokens, .. } = file;
let relative_path = match get_relative_filepath(&vfs, &rootpath, file_id) {
Some(relative_path) => relative_path,
None => continue,
};
let line_index = LineIndex {
index: db.line_index(file_id),
encoding: OffsetEncoding::Utf8,
endings: LineEndings::Unix,
};
let mut doc = scip_types::Document {
relative_path,
language: "rust".to_string(),
..Default::default()
};
tokens.into_iter().for_each(|(range, id)| {
let token = si.tokens.get(id).unwrap();
let mut occurrence = scip_types::Occurrence::default();
occurrence.range = text_range_to_scip_range(&line_index, range);
occurrence.symbol = match tokens_to_symbol.get(&id) {
Some(symbol) => symbol.clone(),
None => {
let symbol = match &token.moniker {
Some(moniker) => moniker_to_symbol(&moniker),
None => new_local_symbol(),
};
let symbol = scip::symbol::format_symbol(symbol);
tokens_to_symbol.insert(id, symbol.clone());
symbol
}
};
if let Some(def) = token.definition {
if def.range == range {
occurrence.symbol_roles |= scip_types::SymbolRole::Definition as i32;
}
if !symbols_emitted.contains(&id) {
symbols_emitted.insert(id);
let mut symbol_info = scip_types::SymbolInformation::default();
symbol_info.symbol = occurrence.symbol.clone();
if let Some(hover) = &token.hover {
if !hover.markup.as_str().is_empty() {
symbol_info.documentation = vec![hover.markup.as_str().to_string()];
}
}
doc.symbols.push(symbol_info)
}
}
doc.occurrences.push(occurrence);
});
if doc.occurrences.is_empty() {
continue;
}
index.documents.push(doc);
}
scip::write_message_to_file("index.scip", index)
.map_err(|err| anyhow::anyhow!("Failed to write scip to file: {}", err))?;
eprintln!("Generating SCIP finished {:?}", now.elapsed());
Ok(())
}
}
fn get_relative_filepath(
vfs: &vfs::Vfs,
rootpath: &vfs::AbsPathBuf,
file_id: ide::FileId,
) -> Option<String> {
Some(vfs.file_path(file_id).as_path()?.strip_prefix(&rootpath)?.as_ref().to_str()?.to_string())
}
// SCIP Ranges have a (very large) optimization that ranges if they are on the same line
// only encode as a vector of [start_line, start_col, end_col].
//
// This transforms a line index into the optimized SCIP Range.
fn text_range_to_scip_range(line_index: &LineIndex, range: TextRange) -> Vec<i32> {
let LineCol { line: start_line, col: start_col } = line_index.index.line_col(range.start());
let LineCol { line: end_line, col: end_col } = line_index.index.line_col(range.end());
if start_line == end_line {
vec![start_line as i32, start_col as i32, end_col as i32]
} else {
vec![start_line as i32, start_col as i32, end_line as i32, end_col as i32]
}
}
fn new_descriptor_str(
name: &str,
suffix: scip_types::descriptor::Suffix,
) -> scip_types::Descriptor {
scip_types::Descriptor {
name: name.to_string(),
disambiguator: "".to_string(),
suffix: suffix.into(),
..Default::default()
}
}
fn new_descriptor(name: Name, suffix: scip_types::descriptor::Suffix) -> scip_types::Descriptor {
let mut name = name.to_string();
if name.contains("'") {
name = format!("`{}`", name);
}
new_descriptor_str(name.as_str(), suffix)
}
/// Loosely based on `def_to_moniker`
///
/// Only returns a Symbol when it's a non-local symbol.
/// So if the visibility isn't outside of a document, then it will return None
fn moniker_to_symbol(moniker: &MonikerResult) -> scip_types::Symbol {
use scip_types::descriptor::Suffix::*;
let package_name = moniker.package_information.name.clone();
let version = moniker.package_information.version.clone();
let descriptors = moniker
.identifier
.description
.iter()
.map(|desc| {
new_descriptor(
desc.name.clone(),
match desc.desc {
MonikerDescriptorKind::Namespace => Namespace,
MonikerDescriptorKind::Type => Type,
MonikerDescriptorKind::Term => Term,
MonikerDescriptorKind::Method => Method,
MonikerDescriptorKind::TypeParameter => TypeParameter,
MonikerDescriptorKind::Parameter => Parameter,
MonikerDescriptorKind::Macro => Macro,
MonikerDescriptorKind::Meta => Meta,
},
)
})
.collect();
scip_types::Symbol {
scheme: "rust-analyzer".into(),
package: Some(scip_types::Package {
manager: "cargo".to_string(),
name: package_name,
version,
..Default::default()
})
.into(),
descriptors,
..Default::default()
}
}
#[cfg(test)]
mod test {
use super::*;
use hir::Semantics;
use ide::{AnalysisHost, FilePosition};
use ide_db::defs::IdentClass;
use ide_db::{base_db::fixture::ChangeFixture, helpers::pick_best_token};
use scip::symbol::format_symbol;
use syntax::SyntaxKind::*;
use syntax::{AstNode, T};
fn position(ra_fixture: &str) -> (AnalysisHost, FilePosition) {
let mut host = AnalysisHost::default();
let change_fixture = ChangeFixture::parse(ra_fixture);
host.raw_database_mut().apply_change(change_fixture.change);
let (file_id, range_or_offset) =
change_fixture.file_position.expect("expected a marker ($0)");
let offset = range_or_offset.expect_offset();
(host, FilePosition { file_id, offset })
}
/// If expected == "", then assert that there are no symbols (this is basically local symbol)
#[track_caller]
fn check_symbol(ra_fixture: &str, expected: &str) {
let (host, position) = position(ra_fixture);
let FilePosition { file_id, offset } = position;
let db = host.raw_database();
let sema = &Semantics::new(db);
let file = sema.parse(file_id).syntax().clone();
let original_token = pick_best_token(file.token_at_offset(offset), |kind| match kind {
IDENT
| INT_NUMBER
| LIFETIME_IDENT
| T![self]
| T![super]
| T![crate]
| T![Self]
| COMMENT => 2,
kind if kind.is_trivia() => 0,
_ => 1,
})
.expect("OK OK");
let navs = sema
.descend_into_macros(original_token.clone())
.into_iter()
.filter_map(|token| {
IdentClass::classify_token(sema, &token).map(IdentClass::definitions).map(|it| {
it.into_iter().flat_map(|def| {
let module = def.module(db).unwrap();
let current_crate = module.krate();
match MonikerResult::from_def(sema.db, def, current_crate) {
Some(moniker_result) => Some(moniker_to_symbol(&moniker_result)),
None => None,
}
})
})
})
.flatten()
.collect::<Vec<_>>();
if expected == "" {
assert_eq!(0, navs.len(), "must have no symbols {:?}", navs);
return;
}
assert_eq!(1, navs.len(), "must have one symbol {:?}", navs);
let res = navs.get(0).unwrap();
let formatted = format_symbol(res.clone());
assert_eq!(formatted, expected);
}
#[test]
fn basic() {
check_symbol(
r#"
//- /lib.rs crate:main deps:foo
use foo::example_mod::func;
fn main() {
func$0();
}
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub mod example_mod {
pub fn func() {}
}
"#,
"rust-analyzer cargo foo 0.1.0 example_mod/func().",
);
}
#[test]
fn symbol_for_trait() {
check_symbol(
r#"
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub mod module {
pub trait MyTrait {
pub fn func$0() {}
}
}
"#,
"rust-analyzer cargo foo 0.1.0 module/MyTrait#func().",
);
}
#[test]
fn symbol_for_trait_constant() {
check_symbol(
r#"
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub mod module {
pub trait MyTrait {
const MY_CONST$0: u8;
}
}
"#,
"rust-analyzer cargo foo 0.1.0 module/MyTrait#MY_CONST.",
);
}
#[test]
fn symbol_for_trait_type() {
check_symbol(
r#"
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub mod module {
pub trait MyTrait {
type MyType$0;
}
}
"#,
// "foo::module::MyTrait::MyType",
"rust-analyzer cargo foo 0.1.0 module/MyTrait#[MyType]",
);
}
#[test]
fn symbol_for_trait_impl_function() {
check_symbol(
r#"
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub mod module {
pub trait MyTrait {
pub fn func() {}
}
struct MyStruct {}
impl MyTrait for MyStruct {
pub fn func$0() {}
}
}
"#,
// "foo::module::MyStruct::MyTrait::func",
"rust-analyzer cargo foo 0.1.0 module/MyStruct#MyTrait#func().",
);
}
#[test]
fn symbol_for_field() {
check_symbol(
r#"
//- /lib.rs crate:main deps:foo
use foo::St;
fn main() {
let x = St { a$0: 2 };
}
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub struct St {
pub a: i32,
}
"#,
"rust-analyzer cargo foo 0.1.0 St#a.",
);
}
#[test]
fn local_symbol_for_local() {
check_symbol(
r#"
//- /lib.rs crate:main deps:foo
use foo::module::func;
fn main() {
func();
}
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub mod module {
pub fn func() {
let x$0 = 2;
}
}
"#,
"",
);
}
}