Add cfg_attr and cleanup code

This commit is contained in:
Wyatt Herkamp 2024-03-09 13:25:56 -05:00
parent f45b080965
commit 79f2651262
8 changed files with 234 additions and 199 deletions

View file

@ -16,6 +16,7 @@ rustc-hash.workspace = true
# locals deps
tt.workspace = true
syntax.workspace = true
[dev-dependencies]
expect-test = "1.4.1"
@ -28,7 +29,6 @@ derive_arbitrary = "1.3.2"
# local deps
mbe.workspace = true
syntax.workspace = true
[lints]
workspace = true
workspace = true

View file

@ -1,70 +0,0 @@
use std::{
fmt::{self, Debug},
slice::Iter as SliceIter,
};
use crate::{cfg_expr::next_cfg_expr, CfgAtom, CfgExpr};
use tt::{Delimiter, SmolStr, Span};
/// Represents a `#[cfg_attr(.., my_attr)]` attribute.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct CfgAttr<S> {
/// Expression in `cfg_attr` attribute.
pub cfg_expr: CfgExpr,
/// Inner attribute.
pub attr: tt::Subtree<S>,
}
impl<S: Clone + Span + Debug> CfgAttr<S> {
/// Parses a sub tree in the form of (cfg_expr, inner_attribute)
pub fn parse(tt: &tt::Subtree<S>) -> Option<CfgAttr<S>> {
let mut iter = tt.token_trees.iter();
let cfg_expr = next_cfg_expr(&mut iter).unwrap_or(CfgExpr::Invalid);
// FIXME: This is probably not the right way to do this
// Get's the span of the next token tree
let first_span = iter.as_slice().first().map(|tt| tt.first_span())?;
let attr = tt::Subtree {
delimiter: Delimiter::invisible_spanned(first_span),
token_trees: iter.cloned().collect(),
};
Some(CfgAttr { cfg_expr, attr: attr })
}
}
#[cfg(test)]
mod tests {
use expect_test::{expect, Expect};
use mbe::{syntax_node_to_token_tree, DummyTestSpanMap, DUMMY};
use syntax::{ast, AstNode};
use crate::{CfgAttr, DnfExpr};
fn check_dnf(input: &str, expected_dnf: Expect, expected_attrs: Expect) {
let source_file = ast::SourceFile::parse(input).ok().unwrap();
let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
let tt = syntax_node_to_token_tree(tt.syntax(), DummyTestSpanMap, DUMMY);
let Some(CfgAttr { cfg_expr, attr }) = CfgAttr::parse(&tt) else {
assert!(false, "failed to parse cfg_attr");
return;
};
let actual = format!("#![cfg({})]", DnfExpr::new(cfg_expr));
expected_dnf.assert_eq(&actual);
let actual_attrs = format!("#![{}]", attr);
expected_attrs.assert_eq(&actual_attrs);
}
#[test]
fn smoke() {
check_dnf(
r#"#![cfg_attr(feature = "nightly", feature(slice_split_at_unchecked))]"#,
expect![[r#"#![cfg(feature = "nightly")]"#]],
expect![r#"#![feature (slice_split_at_unchecked)]"#],
);
check_dnf(
r#"#![cfg_attr(not(feature = "std"), no_std)]"#,
expect![[r#"#![cfg(not(feature = "std"))]"#]],
expect![r#"#![no_std]"#],
);
}
}

View file

@ -2,8 +2,12 @@
//!
//! See: <https://doc.rust-lang.org/reference/conditional-compilation.html#conditional-compilation>
use std::{fmt, slice::Iter as SliceIter};
use std::{fmt, iter::Peekable, slice::Iter as SliceIter};
use syntax::{
ast::{self, Meta},
NodeOrToken,
};
use tt::SmolStr;
/// A simple configuration value passed in from the outside.
@ -47,6 +51,12 @@ impl CfgExpr {
pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
}
/// Parses a `cfg` attribute from the meta
pub fn parse_from_attr_meta(meta: Meta) -> Option<CfgExpr> {
let tt = meta.token_tree()?;
let mut iter = tt.token_trees_and_tokens().skip(1).peekable();
next_cfg_expr_from_syntax(&mut iter)
}
/// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates.
pub fn fold(&self, query: &dyn Fn(&CfgAtom) -> bool) -> Option<bool> {
match self {
@ -62,8 +72,71 @@ impl CfgExpr {
}
}
}
fn next_cfg_expr_from_syntax<I>(iter: &mut Peekable<I>) -> Option<CfgExpr>
where
I: Iterator<Item = NodeOrToken<ast::TokenTree, syntax::SyntaxToken>>,
{
let name = match iter.next() {
None => return None,
Some(NodeOrToken::Token(element)) => match element.kind() {
syntax::T![ident] => SmolStr::new(element.text()),
_ => return Some(CfgExpr::Invalid),
},
Some(_) => return Some(CfgExpr::Invalid),
};
let result = match name.as_str() {
"all" | "any" | "not" => {
let mut preds = Vec::new();
let Some(NodeOrToken::Node(tree)) = iter.next() else {
return Some(CfgExpr::Invalid);
};
let mut tree_iter = tree.token_trees_and_tokens().skip(1).peekable();
while tree_iter
.peek()
.filter(
|element| matches!(element, NodeOrToken::Token(token) if (token.kind() != syntax::T![')'])),
)
.is_some()
{
let pred = next_cfg_expr_from_syntax(&mut tree_iter);
if let Some(pred) = pred {
preds.push(pred);
}
}
let group = match name.as_str() {
"all" => CfgExpr::All(preds),
"any" => CfgExpr::Any(preds),
"not" => CfgExpr::Not(Box::new(preds.pop().unwrap_or(CfgExpr::Invalid))),
_ => unreachable!(),
};
Some(group)
}
_ => match iter.peek() {
Some(NodeOrToken::Token(element)) if (element.kind() == syntax::T![=]) => {
iter.next();
match iter.next() {
Some(NodeOrToken::Token(value_token))
if (value_token.kind() == syntax::SyntaxKind::STRING) =>
{
let value = value_token.text();
let value = SmolStr::new(value.trim_matches('"'));
Some(CfgExpr::Atom(CfgAtom::KeyValue { key: name, value }))
}
_ => None,
}
}
_ => Some(CfgExpr::Atom(CfgAtom::Flag(name))),
},
};
if let Some(NodeOrToken::Token(element)) = iter.peek() {
if element.kind() == syntax::T![,] {
iter.next();
}
}
result
}
pub(crate) fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
let name = match it.next() {
None => return None,
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(),

View file

@ -2,8 +2,7 @@
#![warn(rust_2018_idioms, unused_lifetimes)]
mod cfg_attr;
pub(crate) mod cfg_expr;
mod cfg_expr;
mod dnf;
#[cfg(test)]
mod tests;
@ -13,7 +12,6 @@ use std::fmt;
use rustc_hash::FxHashSet;
use tt::SmolStr;
pub use cfg_attr::CfgAttr;
pub use cfg_expr::{CfgAtom, CfgExpr};
pub use dnf::DnfExpr;

View file

@ -1,7 +1,10 @@
use arbitrary::{Arbitrary, Unstructured};
use expect_test::{expect, Expect};
use mbe::{syntax_node_to_token_tree, DummyTestSpanMap, DUMMY};
use syntax::{ast, AstNode};
use syntax::{
ast::{self, Attr},
AstNode, SourceFile,
};
use crate::{CfgAtom, CfgExpr, CfgOptions, DnfExpr};
@ -12,6 +15,22 @@ fn assert_parse_result(input: &str, expected: CfgExpr) {
let cfg = CfgExpr::parse(&tt);
assert_eq!(cfg, expected);
}
fn check_dnf_from_syntax(input: &str, expect: Expect) {
let parse = SourceFile::parse(input);
let node = match parse.tree().syntax().descendants().find_map(Attr::cast) {
Some(it) => it,
None => {
let node = std::any::type_name::<Attr>();
panic!("Failed to make ast node `{node}` from text {input}")
}
};
let node = node.clone_subtree();
assert_eq!(node.syntax().text_range().start(), 0.into());
let cfg = CfgExpr::parse_from_attr_meta(node.meta().unwrap()).unwrap();
let actual = format!("#![cfg({})]", DnfExpr::new(cfg));
expect.assert_eq(&actual);
}
fn check_dnf(input: &str, expect: Expect) {
let source_file = ast::SourceFile::parse(input).ok().unwrap();
@ -86,6 +105,11 @@ fn smoke() {
check_dnf("#![cfg(not(a))]", expect![[r#"#![cfg(not(a))]"#]]);
}
#[test]
fn cfg_from_attr() {
check_dnf_from_syntax(r#"#[cfg(test)]"#, expect![[r#"#![cfg(test)]"#]]);
check_dnf_from_syntax(r#"#[cfg(not(never))]"#, expect![[r#"#![cfg(not(never))]"#]]);
}
#[test]
fn distribute() {

View file

@ -1,102 +1,65 @@
use std::os::windows::process;
use mbe::syntax_node_to_token_tree;
//! Processes out #[cfg] and #[cfg_attr] attributes from the input for the derive macro
use rustc_hash::FxHashSet;
use syntax::{
ast::{self, Attr, FieldList, HasAttrs, RecordFieldList, TupleFieldList, Variant, VariantList},
ast::{self, Attr, HasAttrs, Meta, VariantList},
AstNode, SyntaxElement, SyntaxNode, T,
};
use tracing::info;
use tracing::{info, warn};
use crate::{db::ExpandDatabase, span_map::SpanMap, MacroCallLoc};
use crate::{db::ExpandDatabase, MacroCallKind, MacroCallLoc};
fn check_cfg_attr(
attr: &Attr,
loc: &MacroCallLoc,
span_map: &SpanMap,
db: &dyn ExpandDatabase,
) -> Option<bool> {
attr.simple_name().as_deref().map(|v| v == "cfg")?;
info!("Checking cfg attr {:?}", attr);
let Some(tt) = attr.token_tree() else {
info!("cfg attr has no expr {:?}", attr);
return Some(true);
};
info!("Checking cfg {:?}", tt);
let tt = tt.syntax().clone();
// Convert to a tt::Subtree
let tt = syntax_node_to_token_tree(&tt, span_map, loc.call_site);
let cfg = cfg::CfgExpr::parse(&tt);
fn check_cfg_attr(attr: &Attr, loc: &MacroCallLoc, db: &dyn ExpandDatabase) -> Option<bool> {
if !attr.simple_name().as_deref().map(|v| v == "cfg")? {
return None;
}
info!("Evaluating cfg {}", attr);
let cfg = cfg::CfgExpr::parse_from_attr_meta(attr.meta()?)?;
info!("Checking cfg {:?}", cfg);
let enabled = db.crate_graph()[loc.krate].cfg_options.check(&cfg) != Some(false);
Some(enabled)
}
enum CfgAttrResult {
Enabled(Attr),
Disabled,
}
fn check_cfg_attr_attr(
attr: &Attr,
loc: &MacroCallLoc,
span_map: &SpanMap,
db: &dyn ExpandDatabase,
) -> Option<CfgAttrResult> {
attr.simple_name().as_deref().map(|v| v == "cfg_attr")?;
info!("Checking cfg_attr attr {:?}", attr);
let Some(tt) = attr.token_tree() else {
info!("cfg_attr attr has no expr {:?}", attr);
fn check_cfg_attr_attr(attr: &Attr, loc: &MacroCallLoc, db: &dyn ExpandDatabase) -> Option<bool> {
if !attr.simple_name().as_deref().map(|v| v == "cfg_attr")? {
return None;
};
info!("Checking cfg_attr {:?}", tt);
let tt = tt.syntax().clone();
// Convert to a tt::Subtree
let tt = syntax_node_to_token_tree(&tt, span_map, loc.call_site);
let cfg = cfg::CfgExpr::parse(&tt);
let enabled = db.crate_graph()[loc.krate].cfg_options.check(&cfg) != Some(false);
if enabled {
// FIXME: Add the internal attribute
Some(CfgAttrResult::Enabled(attr.clone()))
} else {
Some(CfgAttrResult::Disabled)
}
info!("Evaluating cfg_attr {}", attr);
let cfg_expr = cfg::CfgExpr::parse_from_attr_meta(attr.meta()?)?;
info!("Checking cfg_attr {:?}", cfg_expr);
let enabled = db.crate_graph()[loc.krate].cfg_options.check(&cfg_expr) != Some(false);
Some(enabled)
}
fn process_has_attrs_with_possible_comma<I: HasAttrs>(
items: impl Iterator<Item = I>,
loc: &MacroCallLoc,
span_map: &SpanMap,
db: &dyn ExpandDatabase,
res: &mut FxHashSet<SyntaxElement>,
remove: &mut FxHashSet<SyntaxElement>,
) -> Option<()> {
for item in items {
let field_attrs = item.attrs();
'attrs: for attr in field_attrs {
let Some(enabled) = check_cfg_attr(&attr, loc, span_map, db) else {
continue;
};
if enabled {
//FIXME: Should we remove the cfg_attr?
} else {
info!("censoring type {:?}", item.syntax());
res.insert(item.syntax().clone().into());
// We need to remove the , as well
if let Some(comma) = item.syntax().next_sibling_or_token() {
if comma.kind() == T![,] {
res.insert(comma.into());
}
}
break 'attrs;
}
let Some(attr_result) = check_cfg_attr_attr(&attr, loc, span_map, db) else {
continue;
};
match attr_result {
CfgAttrResult::Enabled(attr) => {
//FIXME: Replace the attribute with the internal attribute
}
CfgAttrResult::Disabled => {
if let Some(enabled) = check_cfg_attr(&attr, loc, db) {
// Rustc does not strip the attribute if it is enabled. So we will will leave it
if !enabled {
info!("censoring type {:?}", item.syntax());
res.insert(attr.syntax().clone().into());
remove.insert(item.syntax().clone().into());
// We need to remove the , as well
add_comma(&item, remove);
break 'attrs;
}
};
if let Some(enabled) = check_cfg_attr_attr(&attr, loc, db) {
if enabled {
info!("Removing cfg_attr tokens {:?}", attr);
let meta = attr.meta()?;
let removes_from_cfg_attr = remove_tokens_within_cfg_attr(meta)?;
remove.extend(removes_from_cfg_attr);
} else {
info!("censoring type cfg_attr {:?}", item.syntax());
remove.insert(attr.syntax().clone().into());
continue;
}
}
@ -104,75 +67,130 @@ fn process_has_attrs_with_possible_comma<I: HasAttrs>(
}
Some(())
}
fn remove_tokens_within_cfg_attr(meta: Meta) -> Option<FxHashSet<SyntaxElement>> {
let mut remove: FxHashSet<SyntaxElement> = FxHashSet::default();
info!("Enabling attribute {}", meta);
let meta_path = meta.path()?;
info!("Removing {:?}", meta_path.syntax());
remove.insert(meta_path.syntax().clone().into());
let meta_tt = meta.token_tree()?;
info!("meta_tt {}", meta_tt);
// Remove the left paren
remove.insert(meta_tt.l_paren_token()?.into());
let mut found_comma = false;
for tt in meta_tt.token_trees_and_tokens().skip(1) {
info!("Checking {:?}", tt);
// Check if it is a subtree or a token. If it is a token check if it is a comma. If so, remove it and break.
match tt {
syntax::NodeOrToken::Node(node) => {
// Remove the entire subtree
remove.insert(node.syntax().clone().into());
}
syntax::NodeOrToken::Token(token) => {
if token.kind() == T![,] {
found_comma = true;
remove.insert(token.into());
break;
}
remove.insert(token.into());
}
}
}
if !found_comma {
warn!("No comma found in {}", meta_tt);
return None;
}
// Remove the right paren
remove.insert(meta_tt.r_paren_token()?.into());
Some(remove)
}
fn add_comma(item: &impl AstNode, res: &mut FxHashSet<SyntaxElement>) {
if let Some(comma) = item.syntax().next_sibling_or_token().filter(|it| it.kind() == T![,]) {
res.insert(comma);
}
}
fn process_enum(
variants: VariantList,
loc: &MacroCallLoc,
span_map: &SpanMap,
db: &dyn ExpandDatabase,
res: &mut FxHashSet<SyntaxElement>,
remove: &mut FxHashSet<SyntaxElement>,
) -> Option<()> {
for variant in variants.variants() {
'attrs: for attr in variant.attrs() {
if !check_cfg_attr(&attr, loc, span_map, db)? {
info!("censoring variant {:?}", variant.syntax());
res.insert(variant.syntax().clone().into());
if let Some(comma) = variant.syntax().next_sibling_or_token() {
if comma.kind() == T![,] {
res.insert(comma.into());
}
'variant: for variant in variants.variants() {
for attr in variant.attrs() {
if let Some(enabled) = check_cfg_attr(&attr, loc, db) {
// Rustc does not strip the attribute if it is enabled. So we will will leave it
if !enabled {
info!("censoring type {:?}", variant.syntax());
remove.insert(variant.syntax().clone().into());
// We need to remove the , as well
add_comma(&variant, remove);
continue 'variant;
}
};
if let Some(enabled) = check_cfg_attr_attr(&attr, loc, db) {
if enabled {
info!("Removing cfg_attr tokens {:?}", attr);
let meta = attr.meta()?;
let removes_from_cfg_attr = remove_tokens_within_cfg_attr(meta)?;
remove.extend(removes_from_cfg_attr);
} else {
info!("censoring type cfg_attr {:?}", variant.syntax());
remove.insert(attr.syntax().clone().into());
continue;
}
break 'attrs;
}
}
if let Some(fields) = variant.field_list() {
match fields {
ast::FieldList::RecordFieldList(fields) => {
process_has_attrs_with_possible_comma(fields.fields(), loc, span_map, db, res)?;
process_has_attrs_with_possible_comma(fields.fields(), loc, db, remove)?;
}
ast::FieldList::TupleFieldList(fields) => {
process_has_attrs_with_possible_comma(fields.fields(), loc, span_map, db, res)?;
process_has_attrs_with_possible_comma(fields.fields(), loc, db, remove)?;
}
}
}
}
Some(())
}
/// Handle
pub(crate) fn process_cfg_attrs(
node: &SyntaxNode,
loc: &MacroCallLoc,
span_map: &SpanMap,
db: &dyn ExpandDatabase,
) -> Option<FxHashSet<SyntaxElement>> {
// FIXME: #[cfg_eval] is not implemented. But it is not stable yet
if !matches!(loc.kind, MacroCallKind::Derive { .. }) {
return None;
}
let mut res = FxHashSet::default();
let item = ast::Item::cast(node.clone())?;
match item {
ast::Item::Struct(it) => match it.field_list()? {
ast::FieldList::RecordFieldList(fields) => {
process_has_attrs_with_possible_comma(
fields.fields(),
loc,
span_map,
db,
&mut res,
)?;
process_has_attrs_with_possible_comma(fields.fields(), loc, db, &mut res)?;
}
ast::FieldList::TupleFieldList(fields) => {
process_has_attrs_with_possible_comma(
fields.fields(),
loc,
span_map,
db,
&mut res,
)?;
process_has_attrs_with_possible_comma(fields.fields(), loc, db, &mut res)?;
}
},
ast::Item::Enum(it) => {
process_enum(it.variant_list()?, loc, span_map, db, &mut res)?;
process_enum(it.variant_list()?, loc, db, &mut res)?;
}
// FIXME: Implement for other items
ast::Item::Union(it) => {
process_has_attrs_with_possible_comma(
it.record_field_list()?.fields(),
loc,
db,
&mut res,
)?;
}
// FIXME: Implement for other items if necessary. As we do not support #[cfg_eval] yet, we do not need to implement it for now
_ => {}
}
Some(res)
}

View file

@ -7,10 +7,9 @@ use mbe::{syntax_node_to_token_tree, ValueResult};
use rustc_hash::FxHashSet;
use span::{AstIdMap, SyntaxContextData, SyntaxContextId};
use syntax::{
ast::{self, Attr, HasAttrs},
ast::{self, HasAttrs},
AstNode, Parse, SyntaxElement, SyntaxError, SyntaxNode, SyntaxToken, T,
};
use tracing::info;
use triomphe::Arc;
use crate::{
@ -410,7 +409,8 @@ fn macro_arg(
),
MacroCallKind::Derive { .. } | MacroCallKind::Attr { .. } => {
let censor = censor_for_macro_input(&loc, &syntax);
let censor_cfg = censor_cfg_elements(&syntax, &loc, &map, db);
let censor_cfg =
cfg_process::process_cfg_attrs(&syntax, &loc, db).unwrap_or_default();
let mut fixups = fixup::fixup_syntax(map.as_ref(), &syntax, loc.call_site);
fixups.append.retain(|it, _| match it {
syntax::NodeOrToken::Token(_) => true,
@ -461,14 +461,7 @@ fn macro_arg(
}
}
}
fn censor_cfg_elements(
node: &SyntaxNode,
loc: &MacroCallLoc,
span_map: &SpanMap,
db: &dyn ExpandDatabase,
) -> FxHashSet<SyntaxElement> {
cfg_process::process_cfg_attrs(node, loc, span_map, db).unwrap_or_default()
}
// FIXME: Censoring info should be calculated by the caller! Namely by name resolution
/// Certain macro calls expect some nodes in the input to be preprocessed away, namely:
/// - derives expect all `#[derive(..)]` invocations up to the currently invoked one to be stripped

View file

@ -9,7 +9,6 @@ use syntax::{
SyntaxKind::*,
SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T,
};
use tracing::info;
use tt::{
buffer::{Cursor, TokenBuffer},
Span,