Skip to content

Commit bcf9756

Browse files
ChayimFriedman2ChayimFriedman2
committed
Include a per-token edition in the parser input
Because, as it turns out, this is necessary to determine the correct edition (it is not global). The next commits will make use of it.
1 parent f14bf95 commit bcf9756

File tree

3 files changed

+29
-20
lines changed

3 files changed

+29
-20
lines changed

‎crates/parser/src/input.rs

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
//! See [`Input`].
22
3+
use edition::Edition;
4+
35
use crate::SyntaxKind;
46

57
#[allow(non_camel_case_types)]
@@ -16,6 +18,7 @@ pub struct Input {
1618
kind: Vec<SyntaxKind>,
1719
joint: Vec<bits>,
1820
contextual_kind: Vec<SyntaxKind>,
21+
edition: Vec<Edition>,
1922
}
2023

2124
/// `pub` impl used by callers to create `Tokens`.
@@ -26,15 +29,16 @@ impl Input {
2629
kind: Vec::with_capacity(capacity),
2730
joint: Vec::with_capacity(capacity / size_of::<bits>()),
2831
contextual_kind: Vec::with_capacity(capacity),
32+
edition: Vec::with_capacity(capacity),
2933
}
3034
}
3135
#[inline]
32-
pub fn push(&mut self, kind: SyntaxKind) {
33-
self.push_impl(kind, SyntaxKind::EOF)
36+
pub fn push(&mut self, kind: SyntaxKind, edition: Edition) {
37+
self.push_impl(kind, SyntaxKind::EOF, edition)
3438
}
3539
#[inline]
36-
pub fn push_ident(&mut self, contextual_kind: SyntaxKind) {
37-
self.push_impl(SyntaxKind::IDENT, contextual_kind)
40+
pub fn push_ident(&mut self, contextual_kind: SyntaxKind, edition: Edition) {
41+
self.push_impl(SyntaxKind::IDENT, contextual_kind, edition)
3842
}
3943
/// Sets jointness for the last token we've pushed.
4044
///
@@ -59,13 +63,14 @@ impl Input {
5963
self.joint[idx] |= 1 << b_idx;
6064
}
6165
#[inline]
62-
fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind) {
66+
fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind, edition: Edition) {
6367
let idx = self.len();
6468
if idx % (bits::BITS as usize) == 0 {
6569
self.joint.push(0);
6670
}
6771
self.kind.push(kind);
6872
self.contextual_kind.push(contextual_kind);
73+
self.edition.push(edition);
6974
}
7075
}
7176

@@ -77,6 +82,9 @@ impl Input {
7782
pub(crate) fn contextual_kind(&self, idx: usize) -> SyntaxKind {
7883
self.contextual_kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
7984
}
85+
pub(crate) fn edition(&self, idx: usize) -> Edition {
86+
self.edition[idx]
87+
}
8088
pub(crate) fn is_joint(&self, n: usize) -> bool {
8189
let (idx, b_idx) = self.bit_index(n);
8290
self.joint[idx] & (1 << b_idx) != 0

‎crates/parser/src/shortcuts.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,13 @@ impl LexedStr<'_> {
3838
res.push_ident(
3939
SyntaxKind::from_contextual_keyword(token_text, edition)
4040
.unwrap_or(SyntaxKind::IDENT),
41+
edition,
4142
)
4243
} else {
4344
if was_joint {
4445
res.was_joint();
4546
}
46-
res.push(kind);
47+
res.push(kind, edition);
4748
// Tag the token as joint if it is float with a fractional part
4849
// we use this jointness to inform the parser about what token split
4950
// event to emit when we encounter a float literal in a field access

‎crates/syntax-bridge/src/to_parser_input.rs

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ pub fn to_parser_input<Ctx: Copy + fmt::Debug + PartialEq + Eq + Hash>(
1616

1717
let mut current = buffer.cursor();
1818
let mut syntax_context_to_edition_cache = FxHashMap::default();
19+
let mut ctx_edition =
20+
|ctx| *syntax_context_to_edition_cache.entry(ctx).or_insert_with(|| span_to_edition(ctx));
1921

2022
while !current.eof() {
2123
let tt = current.token_tree();
@@ -25,8 +27,8 @@ pub fn to_parser_input<Ctx: Copy + fmt::Debug + PartialEq + Eq + Hash>(
2527
if punct.char == '\'' {
2628
current.bump();
2729
match current.token_tree() {
28-
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(_ident))) => {
29-
res.push(LIFETIME_IDENT);
30+
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => {
31+
res.push(LIFETIME_IDENT, ctx_edition(ident.span.ctx));
3032
current.bump();
3133
continue;
3234
}
@@ -51,7 +53,7 @@ pub fn to_parser_input<Ctx: Copy + fmt::Debug + PartialEq + Eq + Hash>(
5153
tt::LitKind::CStr | tt::LitKind::CStrRaw(_) => SyntaxKind::C_STRING,
5254
tt::LitKind::Err(_) => SyntaxKind::ERROR,
5355
};
54-
res.push(kind);
56+
res.push(kind, ctx_edition(lit.span.ctx));
5557

5658
if kind == FLOAT_NUMBER && !lit.symbol.as_str().ends_with('.') {
5759
// Tag the token as joint if it is float with a fractional part
@@ -61,28 +63,26 @@ pub fn to_parser_input<Ctx: Copy + fmt::Debug + PartialEq + Eq + Hash>(
6163
}
6264
}
6365
tt::Leaf::Ident(ident) => {
64-
let edition = *syntax_context_to_edition_cache
65-
.entry(ident.span.ctx)
66-
.or_insert_with(|| span_to_edition(ident.span.ctx));
66+
let edition = ctx_edition(ident.span.ctx);
6767
match ident.sym.as_str() {
68-
"_" => res.push(T![_]),
69-
i if i.starts_with('\'') => res.push(LIFETIME_IDENT),
70-
_ if ident.is_raw.yes() => res.push(IDENT),
68+
"_" => res.push(T![_], edition),
69+
i if i.starts_with('\'') => res.push(LIFETIME_IDENT, edition),
70+
_ if ident.is_raw.yes() => res.push(IDENT, edition),
7171
text => match SyntaxKind::from_keyword(text, edition) {
72-
Some(kind) => res.push(kind),
72+
Some(kind) => res.push(kind, edition),
7373
None => {
7474
let contextual_keyword =
7575
SyntaxKind::from_contextual_keyword(text, edition)
7676
.unwrap_or(SyntaxKind::IDENT);
77-
res.push_ident(contextual_keyword);
77+
res.push_ident(contextual_keyword, edition);
7878
}
7979
},
8080
}
8181
}
8282
tt::Leaf::Punct(punct) => {
8383
let kind = SyntaxKind::from_char(punct.char)
8484
.unwrap_or_else(|| panic!("{punct:#?} is not a valid punct"));
85-
res.push(kind);
85+
res.push(kind, ctx_edition(punct.span.ctx));
8686
if punct.spacing == tt::Spacing::Joint {
8787
res.was_joint();
8888
}
@@ -97,7 +97,7 @@ pub fn to_parser_input<Ctx: Copy + fmt::Debug + PartialEq + Eq + Hash>(
9797
tt::DelimiterKind::Bracket => Some(T!['[']),
9898
tt::DelimiterKind::Invisible => None,
9999
} {
100-
res.push(kind);
100+
res.push(kind, ctx_edition(subtree.delimiter.open.ctx));
101101
}
102102
current.bump();
103103
}
@@ -109,7 +109,7 @@ pub fn to_parser_input<Ctx: Copy + fmt::Debug + PartialEq + Eq + Hash>(
109109
tt::DelimiterKind::Bracket => Some(T![']']),
110110
tt::DelimiterKind::Invisible => None,
111111
} {
112-
res.push(kind);
112+
res.push(kind, ctx_edition(subtree.delimiter.close.ctx));
113113
}
114114
}
115115
};

0 commit comments

Comments
 (0)