Skip to content

Commit 819cdf0

Browse files
ChayimFriedman2ChayimFriedman2
committed
Make tt::Literal use one Symbol for the text and the suffix
That shrinks it, which is useless now (it's not the dominant factor), but will become important when we'll implement span compression.
1 parent d3d47b8 commit 819cdf0

17 files changed

Lines changed: 190 additions & 218 deletions

File tree

‎crates/cfg/src/cfg_expr.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,8 @@ fn next_cfg_expr_from_ast(
157157
},
158158
ctx: span::SyntaxContext::root(span::Edition::Edition2015),
159159
};
160-
let literal = tt::token_to_literal(literal.text(), dummy_span).symbol;
160+
let literal =
161+
Symbol::intern(tt::token_to_literal(literal.text(), dummy_span).text());
161162
it.next();
162163
CfgAtom::KeyValue { key: name, value: literal.clone() }.into()
163164
} else {
@@ -211,7 +212,7 @@ fn next_cfg_expr(it: &mut tt::iter::TtIter<'_>) -> Option<CfgExpr> {
211212
Some(tt::TtElement::Leaf(tt::Leaf::Literal(literal))) => {
212213
it.next();
213214
it.next();
214-
CfgAtom::KeyValue { key: name, value: literal.symbol.clone() }.into()
215+
CfgAtom::KeyValue { key: name, value: Symbol::intern(literal.text()) }.into()
215216
}
216217
_ => return Some(CfgExpr::Invalid),
217218
}

‎crates/hir-def/src/item_tree/attrs.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ impl<'attr> AttrQuery<'attr> {
226226
}
227227

228228
#[inline]
229-
pub(crate) fn string_value_with_span(self) -> Option<(&'attr Symbol, span::Span)> {
229+
pub(crate) fn string_value_with_span(self) -> Option<(&'attr str, span::Span)> {
230230
self.attrs().find_map(|attr| attr.string_value_with_span())
231231
}
232232

‎crates/hir-def/src/nameres/collector.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ use hir_expand::{
1717
name::{AsName, Name},
1818
proc_macro::CustomProcMacroExpander,
1919
};
20-
use intern::{Interned, sym};
20+
use intern::{Interned, Symbol, sym};
2121
use itertools::izip;
2222
use la_arena::Idx;
2323
use rustc_hash::{FxHashMap, FxHashSet};
@@ -292,13 +292,13 @@ impl<'db> DefCollector<'db> {
292292
match () {
293293
() if *attr_name == sym::recursion_limit => {
294294
if let Some(limit) = attr.string_value()
295-
&& let Ok(limit) = limit.as_str().parse()
295+
&& let Ok(limit) = limit.parse()
296296
{
297297
crate_data.recursion_limit = Some(limit);
298298
}
299299
}
300300
() if *attr_name == sym::crate_type => {
301-
if attr.string_value() == Some(&sym::proc_dash_macro) {
301+
if attr.string_value() == Some("proc-macro") {
302302
self.is_proc_macro = true;
303303
}
304304
}
@@ -2460,7 +2460,7 @@ impl ModCollector<'_, '_> {
24602460
let name;
24612461
let name = match attrs.by_key(sym::rustc_builtin_macro).string_value_with_span() {
24622462
Some((it, span)) => {
2463-
name = Name::new_symbol(it.clone(), span.ctx);
2463+
name = Name::new_symbol(Symbol::intern(it), span.ctx);
24642464
&name
24652465
}
24662466
None => {

‎crates/hir-expand/src/attrs.rs‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ use arrayvec::ArrayVec;
3535
use base_db::Crate;
3636
use cfg::{CfgExpr, CfgOptions};
3737
use either::Either;
38-
use intern::{Interned, Symbol};
38+
use intern::Interned;
3939
use itertools::Itertools;
4040
use mbe::{DelimiterKind, Punct};
4141
use parser::T;
@@ -417,37 +417,32 @@ impl fmt::Display for AttrInput {
417417

418418
impl Attr {
419419
/// #[path = "string"]
420-
pub fn string_value(&self) -> Option<&Symbol> {
420+
pub fn string_value(&self) -> Option<&str> {
421421
match self.input.as_deref()? {
422-
AttrInput::Literal(tt::Literal {
423-
symbol: text,
424-
kind: tt::LitKind::Str | tt::LitKind::StrRaw(_),
425-
..
426-
}) => Some(text),
422+
AttrInput::Literal(
423+
lit @ tt::Literal { kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), .. },
424+
) => Some(lit.text()),
427425
_ => None,
428426
}
429427
}
430428

431429
/// #[path = "string"]
432-
pub fn string_value_with_span(&self) -> Option<(&Symbol, span::Span)> {
430+
pub fn string_value_with_span(&self) -> Option<(&str, span::Span)> {
433431
match self.input.as_deref()? {
434-
AttrInput::Literal(tt::Literal {
435-
symbol: text,
436-
kind: tt::LitKind::Str | tt::LitKind::StrRaw(_),
437-
span,
438-
suffix: _,
439-
}) => Some((text, *span)),
432+
AttrInput::Literal(
433+
lit @ tt::Literal { kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), span, .. },
434+
) => Some((lit.text(), *span)),
440435
_ => None,
441436
}
442437
}
443438

444439
pub fn string_value_unescape(&self) -> Option<Cow<'_, str>> {
445440
match self.input.as_deref()? {
446-
AttrInput::Literal(tt::Literal {
447-
symbol: text, kind: tt::LitKind::StrRaw(_), ..
448-
}) => Some(Cow::Borrowed(text.as_str())),
449-
AttrInput::Literal(tt::Literal { symbol: text, kind: tt::LitKind::Str, .. }) => {
450-
unescape(text.as_str())
441+
AttrInput::Literal(lit @ tt::Literal { kind: tt::LitKind::StrRaw(_), .. }) => {
442+
Some(Cow::Borrowed(lit.text()))
443+
}
444+
AttrInput::Literal(lit @ tt::Literal { kind: tt::LitKind::Str, .. }) => {
445+
unescape(lit.text())
451446
}
452447
_ => None,
453448
}

‎crates/hir-expand/src/builtin/fn_macro.rs‎

Lines changed: 42 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
//! Builtin macro
22
3+
use std::borrow::Cow;
4+
35
use base_db::AnchoredPath;
46
use cfg::CfgExpr;
57
use either::Either;
@@ -13,7 +15,7 @@ use span::{Edition, FileId, Span};
1315
use stdx::format_to;
1416
use syntax::{
1517
format_smolstr,
16-
unescape::{unescape_byte, unescape_char, unescape_str},
18+
unescape::{unescape_byte, unescape_char},
1719
};
1820
use syntax_bridge::syntax_node_to_token_tree;
1921

@@ -177,12 +179,7 @@ fn line_expand(
177179
// not incremental
178180
ExpandResult::ok(tt::TopSubtree::invisible_from_leaves(
179181
span,
180-
[tt::Leaf::Literal(tt::Literal {
181-
symbol: sym::INTEGER_0,
182-
span,
183-
kind: tt::LitKind::Integer,
184-
suffix: Some(sym::u32),
185-
})],
182+
[tt::Leaf::Literal(tt::Literal::new("0", span, tt::LitKind::Integer, "u32"))],
186183
))
187184
}
188185

@@ -303,7 +300,8 @@ fn format_args_nl_expand(
303300
mut lit @ tt::Literal { kind: tt::LitKind::Str, .. },
304301
))) = lit
305302
{
306-
lit.symbol = Symbol::intern(&format_smolstr!("{}\\n", lit.symbol.as_str()));
303+
let (text, suffix) = lit.text_and_suffix();
304+
lit.text_and_suffix = Symbol::intern(&format_smolstr!("{text}\\n{suffix}"));
307305
tt.set_token(1, lit.into());
308306
}
309307
ExpandResult::ok(quote! {span =>
@@ -521,14 +519,11 @@ fn compile_error_expand(
521519
let err = match tt.iter().collect_array() {
522520
Some(
523521
[
524-
tt::TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
525-
symbol: text,
526-
span: _,
527-
kind: tt::LitKind::Str | tt::LitKind::StrRaw(_),
528-
suffix: _,
529-
})),
522+
tt::TtElement::Leaf(tt::Leaf::Literal(
523+
lit @ tt::Literal { kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), .. },
524+
)),
530525
],
531-
) => ExpandError::other(span, Box::from(unescape_symbol(&text).as_str())),
526+
) => ExpandError::other(span, Box::from(unescape_str(lit.text()))),
532527
_ => ExpandError::other(span, "`compile_error!` argument must be a string"),
533528
};
534529

@@ -569,20 +564,20 @@ fn concat_expand(
569564
// as-is.
570565
match it.kind {
571566
tt::LitKind::Char => {
572-
if let Ok(c) = unescape_char(it.symbol.as_str()) {
567+
if let Ok(c) = unescape_char(it.text()) {
573568
text.push(c);
574569
}
575570
record_span(it.span);
576571
}
577572
tt::LitKind::Integer | tt::LitKind::Float => {
578-
format_to!(text, "{}", it.symbol.as_str())
573+
format_to!(text, "{}", it.text())
579574
}
580575
tt::LitKind::Str => {
581-
text.push_str(unescape_symbol(&it.symbol).as_str());
576+
text.push_str(&unescape_str(it.text()));
582577
record_span(it.span);
583578
}
584579
tt::LitKind::StrRaw(_) => {
585-
format_to!(text, "{}", it.symbol.as_str());
580+
format_to!(text, "{}", it.text());
586581
record_span(it.span);
587582
}
588583
tt::LitKind::Byte
@@ -620,7 +615,7 @@ fn concat_expand(
620615
TtElement::Leaf(tt::Leaf::Literal(it))
621616
if matches!(it.kind, tt::LitKind::Integer | tt::LitKind::Float) =>
622617
{
623-
format_to!(text, "-{}", it.symbol.as_str());
618+
format_to!(text, "-{}", it.text());
624619
record_span(punct.span.cover(it.span));
625620
}
626621
_ => {
@@ -658,26 +653,22 @@ fn concat_bytes_expand(
658653
};
659654
for (i, t) in tt.iter().enumerate() {
660655
match t {
661-
TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
662-
symbol: text,
663-
span,
664-
kind,
665-
suffix: _,
666-
})) => {
656+
TtElement::Leaf(tt::Leaf::Literal(lit @ tt::Literal { span, kind, .. })) => {
657+
let text = lit.text();
667658
record_span(span);
668659
match kind {
669660
tt::LitKind::Byte => {
670-
if let Ok(b) = unescape_byte(text.as_str()) {
661+
if let Ok(b) = unescape_byte(text) {
671662
bytes.extend(
672663
b.escape_ascii().filter_map(|it| char::from_u32(it as u32)),
673664
);
674665
}
675666
}
676667
tt::LitKind::ByteStr => {
677-
bytes.push_str(text.as_str());
668+
bytes.push_str(text);
678669
}
679670
tt::LitKind::ByteStrRaw(_) => {
680-
bytes.extend(text.as_str().escape_debug());
671+
bytes.extend(text.escape_debug());
681672
}
682673
_ => {
683674
err.get_or_insert(ExpandError::other(span, "unexpected token"));
@@ -706,12 +697,7 @@ fn concat_bytes_expand(
706697
ExpandResult {
707698
value: tt::TopSubtree::invisible_from_leaves(
708699
span,
709-
[tt::Leaf::Literal(tt::Literal {
710-
symbol: Symbol::intern(&bytes),
711-
span,
712-
kind: tt::LitKind::ByteStr,
713-
suffix: None,
714-
})],
700+
[tt::Leaf::Literal(tt::Literal::new_no_suffix(&bytes, span, tt::LitKind::ByteStr))],
715701
),
716702
err,
717703
}
@@ -725,25 +711,19 @@ fn concat_bytes_expand_subtree(
725711
) -> Result<(), ExpandError> {
726712
for (ti, tt) in tree_iter.enumerate() {
727713
match tt {
728-
TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
729-
symbol: text,
730-
span,
731-
kind: tt::LitKind::Byte,
732-
suffix: _,
733-
})) => {
734-
if let Ok(b) = unescape_byte(text.as_str()) {
714+
TtElement::Leaf(tt::Leaf::Literal(
715+
lit @ tt::Literal { span, kind: tt::LitKind::Byte, .. },
716+
)) => {
717+
if let Ok(b) = unescape_byte(lit.text()) {
735718
bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32)));
736719
}
737720
record_span(span);
738721
}
739-
TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
740-
symbol: text,
741-
span,
742-
kind: tt::LitKind::Integer,
743-
suffix: _,
744-
})) => {
722+
TtElement::Leaf(tt::Leaf::Literal(
723+
lit @ tt::Literal { span, kind: tt::LitKind::Integer, .. },
724+
)) => {
745725
record_span(span);
746-
if let Ok(b) = text.as_str().parse::<u8>() {
726+
if let Ok(b) = lit.text().parse::<u8>() {
747727
bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32)));
748728
}
749729
}
@@ -792,18 +772,16 @@ fn parse_string(tt: &tt::TopSubtree) -> Result<(Symbol, Span), ExpandError> {
792772
}
793773

794774
match tt {
795-
TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
796-
symbol: text,
775+
TtElement::Leaf(tt::Leaf::Literal(lit @ tt::Literal {
797776
span,
798777
kind: tt::LitKind::Str,
799-
suffix: _,
800-
})) => Ok((unescape_symbol(&text), span)),
801-
TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
802-
symbol: text,
778+
..
779+
})) => Ok((Symbol::intern(&unescape_str(lit.text())), span)),
780+
TtElement::Leaf(tt::Leaf::Literal(lit @ tt::Literal {
803781
span,
804782
kind: tt::LitKind::StrRaw(_),
805-
suffix: _,
806-
})) => Ok((text.clone(), span)),
783+
..
784+
})) => Ok((Symbol::intern(lit.text()), span)),
807785
TtElement::Leaf(l) => Err(*l.span()),
808786
TtElement::Subtree(tt, _) => Err(tt.delimiter.open.cover(tt.delimiter.close)),
809787
}
@@ -855,10 +833,10 @@ fn include_bytes_expand(
855833
let res = tt::TopSubtree::invisible_from_leaves(
856834
span,
857835
[tt::Leaf::Literal(tt::Literal {
858-
symbol: Symbol::empty(),
836+
text_and_suffix: Symbol::empty(),
859837
span,
860838
kind: tt::LitKind::ByteStrRaw(1),
861-
suffix: None,
839+
suffix_len: 0,
862840
})],
863841
);
864842
ExpandResult::ok(res)
@@ -979,17 +957,16 @@ fn quote_expand(
979957
)
980958
}
981959

982-
fn unescape_symbol(s: &Symbol) -> Symbol {
983-
if s.as_str().contains('\\') {
984-
let s = s.as_str();
960+
fn unescape_str(s: &str) -> Cow<'_, str> {
961+
if s.contains('\\') {
985962
let mut buf = String::with_capacity(s.len());
986-
unescape_str(s, |_, c| {
963+
syntax::unescape::unescape_str(s, |_, c| {
987964
if let Ok(c) = c {
988965
buf.push(c)
989966
}
990967
});
991-
Symbol::intern(&buf)
968+
Cow::Owned(buf)
992969
} else {
993-
s.clone()
970+
Cow::Borrowed(s)
994971
}
995972
}

‎crates/hir-expand/src/builtin/quote.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -199,16 +199,16 @@ impl<T: ToTokenTree + Clone> ToTokenTree for &T {
199199
}
200200

201201
impl_to_to_tokentrees! {
202-
span: u32 => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } };
203-
span: usize => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } };
204-
span: i32 => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } };
202+
span: u32 => self { crate::tt::Literal{text_and_suffix: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix_len: 0 } };
203+
span: usize => self { crate::tt::Literal{text_and_suffix: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix_len: 0 } };
204+
span: i32 => self { crate::tt::Literal{text_and_suffix: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix_len: 0 } };
205205
span: bool => self { crate::tt::Ident{sym: if self { sym::true_ } else { sym::false_ }, span, is_raw: tt::IdentIsRaw::No } };
206206
_span: crate::tt::Leaf => self { self };
207207
_span: crate::tt::Literal => self { self };
208208
_span: crate::tt::Ident => self { self };
209209
_span: crate::tt::Punct => self { self };
210-
span: &str => self { crate::tt::Literal{symbol: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix: None }};
211-
span: String => self { crate::tt::Literal{symbol: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix: None }};
210+
span: &str => self { crate::tt::Literal{text_and_suffix: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix_len: 0 }};
211+
span: String => self { crate::tt::Literal{text_and_suffix: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix_len: 0 }};
212212
span: Name => self {
213213
let (is_raw, s) = IdentIsRaw::split_from_symbol(self.as_str());
214214
crate::tt::Ident{sym: Symbol::intern(s), span, is_raw }

‎crates/hir-expand/src/fixup.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,7 @@ mod tests {
415415
// `TokenTree`s, see the last assertion in `check()`.
416416
fn check_leaf_eq(a: &tt::Leaf, b: &tt::Leaf) -> bool {
417417
match (a, b) {
418-
(tt::Leaf::Literal(a), tt::Leaf::Literal(b)) => a.symbol == b.symbol,
418+
(tt::Leaf::Literal(a), tt::Leaf::Literal(b)) => a.text_and_suffix == b.text_and_suffix,
419419
(tt::Leaf::Punct(a), tt::Leaf::Punct(b)) => a.char == b.char,
420420
(tt::Leaf::Ident(a), tt::Leaf::Ident(b)) => a.sym == b.sym,
421421
_ => false,

0 commit comments

Comments
 (0)