From e049bd21e2dbd6a8d43248ad713b5d345addea0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 2 May 2025 11:39:18 +0100 Subject: [PATCH 1/3] double slash in paths is a syntax error --- src/tokenizer.rs | 2 ++ test_data/parser/error/path_double_slash.expect | 5 +++++ test_data/parser/error/path_double_slash.nix | 1 + 3 files changed, 8 insertions(+) create mode 100644 test_data/parser/error/path_double_slash.expect create mode 100644 test_data/parser/error/path_double_slash.nix diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 6cddb7f..4fd1b5d 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -172,6 +172,8 @@ impl Tokenizer<'_> { self.ctx.push(Context::InterpolStart); } else if self.str_since(past).ends_with('/') { return TOKEN_ERROR; + } else if self.str_since(past).contains("//") { + return TOKEN_ERROR; } else { self.pop_ctx(Context::Path); } diff --git a/test_data/parser/error/path_double_slash.expect b/test_data/parser/error/path_double_slash.expect new file mode 100644 index 0000000..bd1c5eb --- /dev/null +++ b/test_data/parser/error/path_double_slash.expect @@ -0,0 +1,5 @@ +error: unexpected TOKEN_ERROR at 0..9, wanted any of [TOKEN_L_PAREN, TOKEN_REC, TOKEN_L_BRACE, TOKEN_L_BRACK, TOKEN_STRING_START, TOKEN_IDENT] +NODE_ROOT@0..9 + NODE_ERROR@0..9 + TOKEN_ERROR@0..9 "/foo//bar" + diff --git a/test_data/parser/error/path_double_slash.nix b/test_data/parser/error/path_double_slash.nix new file mode 100644 index 0000000..483fd94 --- /dev/null +++ b/test_data/parser/error/path_double_slash.nix @@ -0,0 +1 @@ +/foo//bar \ No newline at end of file From 588d0b575ccafbbdd957a95dc8e4d7ae7eeead59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Sun, 20 Apr 2025 08:38:24 +0100 Subject: [PATCH 2/3] Distinguish different types of Nix paths at the type level - Added distinct token and node types for different path varieties: - PathAbs: Absolute paths starting with `/` (e.g., `/nix/store/...`) - PathRel: Relative paths (e.g., `./foo`, `foo/bar`) - PathHome: Home-relative paths starting with `~/` (e.g., `~/foo/bar`) - PathSearch: NIX_PATH expressions (e.g., ``) - Created `InterpolatablePath` trait for paths that support interpolation (all except search paths) - Added helper methods to path content for easier path type checking and access through `match_path`, `is_search`, `is_interpolatable`, etc. This allows code to determine what kind of path it's dealing with without having to re-parse the path string, improving type safety, correctness, and making path-specific operations more explicit. --- flake.lock | 30 ++- src/ast.rs | 2 + src/ast/nodes.rs | 20 +- src/ast/path_util.rs | 201 +++++++++++++++--- src/ast/tokens.rs | 55 ++++- src/kinds.rs | 23 +- src/parser.rs | 22 +- src/tokenizer.rs | 120 ++++++++--- .../error/path_interp_no_separator.expect | 2 +- .../error/path_interp_trailing_slash.expect | 4 +- .../parser/error/path_store_interp.expect | 6 +- .../parser/success/import_nixpkgs.expect | 4 +- test_data/parser/success/path.expect | 16 +- test_data/parser/success/path_interp.expect | 28 +-- .../success/path_interp_no_prefix.expect | 4 +- .../parser/success/path_no_newline.expect | 4 +- .../parser/success/with-import-let-in.expect | 4 +- .../error/path_interp_trailing_slash.expect | 2 +- .../tokenizer/success/path_absolute.expect | 2 +- test_data/tokenizer/success/path_home.expect | 2 +- .../tokenizer/success/path_interp.expect | 2 +- .../success/path_interp_apply.expect | 2 +- .../success/path_interp_multiple.expect | 2 +- .../success/path_interp_multiple2.expect | 4 +- .../success/path_interp_then_plain.expect | 4 +- .../tokenizer/success/path_isnt_math.expect | 2 +- .../tokenizer/success/path_no_newline.expect | 2 +- .../tokenizer/success/path_relative.expect | 2 +- .../success/path_relative_prefix.expect | 2 +- test_data/tokenizer/success/path_store.expect | 2 +- .../tokenizer/success/path_underscore.expect | 2 +- 31 files changed, 450 insertions(+), 127 deletions(-) diff --git a/flake.lock b/flake.lock index c86c6cf..139710f 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1656461576, - "narHash": "sha256-rlmmw6lIlkMQIiB+NsnO8wQYWTfle8TA41UREPLP5VY=", + "lastModified": 1744932701, + "narHash": "sha256-fusHbZCyv126cyArUwwKrLdCkgVAIaa/fQJYFlCEqiU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "cf3ab54b4afe2b7477faa1dd0b65bf74c055d70c", + "rev": "b024ced1aac25639f8ca8fdfc2f8c4fbd66c48ef", "type": "github" }, "original": { @@ -38,13 +38,31 @@ "utils": "utils" } }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, "utils": { + "inputs": { + "systems": "systems" + }, "locked": { - "lastModified": 1656065134, - "narHash": "sha256-oc6E6ByIw3oJaIyc67maaFcnjYOz1mMcOtHxbEf9NwQ=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "bee6a7250dd1b01844a2de7e02e4df7d8a0a206c", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { diff --git a/src/ast.rs b/src/ast.rs index 6698ef4..bedc3d9 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -5,6 +5,8 @@ mod interpol; mod nodes; mod operators; mod path_util; + +pub use path_util::Path; mod str_util; mod tokens; diff --git a/src/ast/nodes.rs b/src/ast/nodes.rs index ed9ec0a..7dff2f0 100644 --- a/src/ast/nodes.rs +++ b/src/ast/nodes.rs @@ -176,7 +176,10 @@ node! { IfElse, Select, Str, - Path, + PathAbs, + PathRel, + PathHome, + PathSearch, Literal, Lambda, LegacyLet, @@ -279,7 +282,20 @@ impl InheritFrom { tg! { r_paren_token, ')' } } -node! { #[from(NODE_PATH)] struct Path; } +node! { #[from(NODE_PATH_ABS)] struct PathAbs; } +node! { #[from(NODE_PATH_REL)] struct PathRel; } +node! { #[from(NODE_PATH_HOME)] struct PathHome; } +node! { #[from(NODE_PATH_SEARCH)] struct PathSearch; } + +node! { + #[from( + PathAbs, + PathRel, + PathHome, + PathSearch, + )] + enum Path; +} node! { #[from(NODE_STRING)] struct Str; } diff --git a/src/ast/path_util.rs b/src/ast/path_util.rs index c8b3f95..bfd059b 100644 --- a/src/ast/path_util.rs +++ b/src/ast/path_util.rs @@ -1,21 +1,88 @@ -use crate::{ast::AstToken, kinds::SyntaxKind::*}; +use crate::kinds::SyntaxKind::{TOKEN_PATH_ABS, TOKEN_PATH_HOME, TOKEN_PATH_REL}; + use rowan::{ast::AstNode as OtherAstNode, NodeOrToken}; +pub use super::nodes::Path; +use super::{ + nodes::{PathAbs, PathHome, PathRel, PathSearch}, + AstToken, InterpolPart, PathContent, +}; use crate::ast; -use super::{InterpolPart, PathContent}; - -impl ast::nodes::Path { - pub fn parts(&self) -> impl Iterator> { - self.syntax().children_with_tokens().map(|child| match child { +fn extract_path_parts(node: &T) -> Vec> { + node.syntax() + .children_with_tokens() + .map(|child| match child { NodeOrToken::Token(token) => { - assert_eq!(token.kind(), TOKEN_PATH); + debug_assert!(matches!( + token.kind(), + TOKEN_PATH_ABS | TOKEN_PATH_REL | TOKEN_PATH_HOME + )); InterpolPart::Literal(PathContent::cast(token).unwrap()) } NodeOrToken::Node(node) => { InterpolPart::Interpolation(ast::Interpol::cast(node.clone()).unwrap()) } }) + .collect() +} + +// Direct methods for interpolatable path types +impl PathAbs { + pub fn parts(&self) -> Vec> { + extract_path_parts(self) + } +} + +impl PathRel { + pub fn parts(&self) -> Vec> { + extract_path_parts(self) + } +} + +impl PathHome { + pub fn parts(&self) -> Vec> { + extract_path_parts(self) + } +} + +// Direct methods for search path +impl PathSearch { + /// Get the content of a search path + pub fn content(&self) -> Option { + self.syntax() + .children_with_tokens() + .filter_map(|child| child.into_token().and_then(PathContent::cast)) + .next() + } +} + +/// Extension methods for the Path enum +impl Path { + /// Get parts from any path type in a unified way + pub fn parts(&self) -> Vec> { + match self { + // For interpolatable paths, get their parts + Path::PathAbs(p) => p.parts(), + Path::PathRel(p) => p.parts(), + Path::PathHome(p) => p.parts(), + // For search paths, return a single literal component if content exists + Path::PathSearch(p) => { + if let Some(content) = p.content() { + vec![InterpolPart::Literal(content)] + } else { + vec![] + } + } + } + } + + pub fn is_search(&self) -> bool { + matches!(self, Path::PathSearch(_)) + } + + pub fn is_interpolatable(&self) -> bool { + !self.is_search() } } @@ -23,43 +90,115 @@ impl ast::nodes::Path { mod tests { use rowan::ast::AstNode; + use super::InterpolPart; use crate::{ - ast::{self, AstToken, InterpolPart, PathContent}, + ast::{self, Path}, Root, }; #[test] - fn parts() { - fn assert_eq_ast_ctn(it: &mut dyn Iterator>, x: &str) { - let tmp = it.next().expect("unexpected EOF"); - if let InterpolPart::Interpolation(astn) = tmp { - assert_eq!(astn.expr().unwrap().syntax().to_string(), x); - } else { - unreachable!("unexpected literal {:?}", tmp); + fn test_path_types() { + // Absolute path + let inp = "/foo/bar"; + let expr = Root::parse(inp).ok().unwrap().expr().unwrap(); + if let ast::Expr::PathAbs(p) = expr { + let path = Path::cast(p.syntax().clone()).unwrap(); + assert!(path.is_interpolatable()); + assert!(!path.is_search()); + } + + // Search path + let inp = ""; + let expr = Root::parse(inp).ok().unwrap().expr().unwrap(); + if let ast::Expr::PathSearch(p) = expr { + let path = Path::cast(p.syntax().clone()).unwrap(); + assert!(!path.is_interpolatable()); + assert!(path.is_search()); + } + } + + #[test] + fn test_parts() { + // Test parts with absolute path + let inp = "/foo/bar"; + let expr = Root::parse(inp).ok().unwrap().expr().unwrap(); + if let ast::Expr::PathAbs(p) = expr { + let path = Path::cast(p.syntax().clone()).unwrap(); + + let parts = path.parts(); + assert_eq!(parts.len(), 1); + + match &parts[0] { + InterpolPart::Literal(content) => { + assert_eq!(content.text(), "/foo/bar"); + } + _ => panic!("Expected literal part"), } } - fn assert_eq_lit(it: &mut dyn Iterator>, x: &str) { - let tmp = it.next().expect("unexpected EOF"); - if let InterpolPart::Literal(astn) = tmp { - assert_eq!(astn.syntax().text(), x); - } else { - unreachable!("unexpected interpol {:?}", tmp); + // Test parts with interpolated path + let inp = r#"./a/${"hello"}"#; + let expr = Root::parse(inp).ok().unwrap().expr().unwrap(); + if let ast::Expr::PathRel(p) = expr { + let path = Path::cast(p.syntax().clone()).unwrap(); + + let parts = path.parts(); + assert_eq!(parts.len(), 2); + + match &parts[0] { + InterpolPart::Literal(content) => { + assert_eq!(content.text(), "./a/"); + } + _ => panic!("Expected literal part"), + } + + match &parts[1] { + InterpolPart::Interpolation(_) => {} // Success + _ => panic!("Expected interpolation part"), } } - let inp = r#"./a/b/${"c"}/${d}/e/f"#; + // Test parts with search path + let inp = ""; let expr = Root::parse(inp).ok().unwrap().expr().unwrap(); - match expr { - ast::Expr::Path(p) => { - let mut it = p.parts(); - assert_eq_lit(&mut it, "./a/b/"); - assert_eq_ast_ctn(&mut it, "\"c\""); - assert_eq_lit(&mut it, "/"); - assert_eq_ast_ctn(&mut it, "d"); - assert_eq_lit(&mut it, "/e/f"); + if let ast::Expr::PathSearch(p) = expr { + let path = Path::cast(p.syntax().clone()).unwrap(); + + let parts = path.parts(); + assert_eq!(parts.len(), 1); + + match &parts[0] { + InterpolPart::Literal(content) => { + assert_eq!(content.text(), ""); + } + _ => panic!("Expected literal part"), + } + } + } + + #[test] + fn direct_method_usage() { + // Test direct parts() method on PathAbs + let inp = "/foo/bar"; + let expr = Root::parse(inp).ok().unwrap().expr().unwrap(); + if let ast::Expr::PathAbs(p) = expr { + let parts = p.parts(); + assert_eq!(parts.len(), 1); + + match &parts[0] { + InterpolPart::Literal(content) => { + assert_eq!(content.text(), "/foo/bar"); + } + _ => panic!("Expected literal part"), } - _ => unreachable!(), + } + + // Test direct content() method on PathSearch + let inp = ""; + let expr = Root::parse(inp).ok().unwrap().expr().unwrap(); + if let ast::Expr::PathSearch(p) = expr { + let content = p.content().expect("Expected content"); + assert_eq!(content.text(), ""); } } } diff --git a/src/ast/tokens.rs b/src/ast/tokens.rs index 362812f..f0fd8f7 100644 --- a/src/ast/tokens.rs +++ b/src/ast/tokens.rs @@ -114,7 +114,60 @@ impl Integer { } } -token! { #[from(TOKEN_PATH)] struct PathContent; } +// A literal part of a path (absolute / relative / home / search) without +// distinguishing which concrete flavour it was taken from. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct PathContent(pub(super) SyntaxToken); + +impl PathContent { + /// Returns the path as a string + pub fn text(&self) -> &str { + self.syntax().text() + } + + /// Returns the kind of path token + pub fn path_kind(&self) -> SyntaxKind { + self.syntax().kind() + } + + /// Returns true if this is an absolute path + pub fn is_absolute(&self) -> bool { + self.path_kind() == TOKEN_PATH_ABS + } + + /// Returns true if this is a relative path + pub fn is_relative(&self) -> bool { + self.path_kind() == TOKEN_PATH_REL + } + + /// Returns true if this is a home-relative path + pub fn is_home(&self) -> bool { + self.path_kind() == TOKEN_PATH_HOME + } + + /// Returns true if this is a search path + pub fn is_search(&self) -> bool { + self.path_kind() == TOKEN_PATH_SEARCH + } +} + +impl AstToken for PathContent { + fn can_cast(kind: SyntaxKind) -> bool { + matches!(kind, TOKEN_PATH_ABS | TOKEN_PATH_REL | TOKEN_PATH_HOME | TOKEN_PATH_SEARCH) + } + + fn cast(from: SyntaxToken) -> Option { + if Self::can_cast(from.kind()) { + Some(Self(from)) + } else { + None + } + } + + fn syntax(&self) -> &SyntaxToken { + &self.0 + } +} token! { #[from(TOKEN_STRING_CONTENT)] struct StrContent; } diff --git a/src/kinds.rs b/src/kinds.rs index 9ccd26c..ec62f5d 100644 --- a/src/kinds.rs +++ b/src/kinds.rs @@ -63,7 +63,12 @@ pub enum SyntaxKind { TOKEN_INTEGER, TOKEN_INTERPOL_END, TOKEN_INTERPOL_START, - TOKEN_PATH, + // Path tokens (distinct kinds) + TOKEN_PATH_ABS, // /foo/bar, /nix/store/… + TOKEN_PATH_REL, // foo/bar, ./foo, ../bar + TOKEN_PATH_HOME, // ~/foo/bar + TOKEN_PATH_SEARCH, // + TOKEN_URI, TOKEN_STRING_CONTENT, TOKEN_STRING_END, @@ -98,7 +103,10 @@ pub enum SyntaxKind { NODE_UNARY_OP, NODE_LITERAL, NODE_WITH, - NODE_PATH, + NODE_PATH_ABS, + NODE_PATH_REL, + NODE_PATH_HOME, + NODE_PATH_SEARCH, // Attrpath existence check: foo ? bar.${baz}."bux" NODE_HAS_ATTR, @@ -110,7 +118,16 @@ use SyntaxKind::*; impl SyntaxKind { /// Returns true if this token is a literal, such as an integer or a string pub fn is_literal(self) -> bool { - matches!(self, TOKEN_FLOAT | TOKEN_INTEGER | TOKEN_PATH | TOKEN_URI) + matches!( + self, + TOKEN_FLOAT + | TOKEN_INTEGER + | TOKEN_PATH_ABS + | TOKEN_PATH_REL + | TOKEN_PATH_HOME + | TOKEN_PATH_SEARCH + | TOKEN_URI + ) } /// Returns true if this token should be used as a function argument. diff --git a/src/parser.rs b/src/parser.rs index 895af92..686d2a5 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -485,14 +485,28 @@ where self.finish_node(); } TOKEN_STRING_START => self.parse_string(), - TOKEN_PATH => { - self.start_node(NODE_PATH); + TOKEN_PATH_ABS | TOKEN_PATH_REL | TOKEN_PATH_HOME | TOKEN_PATH_SEARCH => { + let node_kind = match self.peek().unwrap() { + TOKEN_PATH_ABS => NODE_PATH_ABS, + TOKEN_PATH_REL => NODE_PATH_REL, + TOKEN_PATH_HOME => NODE_PATH_HOME, + TOKEN_PATH_SEARCH => NODE_PATH_SEARCH, + _ => unreachable!(), + }; + self.start_node(node_kind); self.bump(); - let is_complex_path = self.peek().map_or(false, |t| t == TOKEN_INTERPOL_START); + + // Search paths () don't support interpolation + let is_search_path = node_kind == NODE_PATH_SEARCH; + let is_complex_path = + !is_search_path && self.peek().map_or(false, |t| t == TOKEN_INTERPOL_START); + if is_complex_path { loop { match self.peek_raw().map(|(t, _)| t) { - Some(TOKEN_PATH) => self.bump(), + Some(TOKEN_PATH_ABS) | Some(TOKEN_PATH_REL) | Some(TOKEN_PATH_HOME) => { + self.bump() + } Some(TOKEN_INTERPOL_START) => { self.start_node(NODE_INTERPOL); self.bump(); diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 4fd1b5d..88c412d 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -5,8 +5,10 @@ use crate::SyntaxKind::{self, *}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum IdentType { Ident, - Path, - Store, + PathRel, + PathAbs, + PathHome, + PathSearch, Uri, } @@ -27,7 +29,7 @@ enum Context { StringEnd, Interpol { brackets: u32 }, InterpolStart, - Path, + Path(SyntaxKind), } #[derive(Clone, Copy)] @@ -93,7 +95,10 @@ impl Tokenizer<'_> { } fn pop_ctx(&mut self, ctx: Context) { - debug_assert_eq!(self.ctx.last(), Some(&ctx)); + debug_assert!(self + .ctx + .last() + .map_or(false, |c| std::mem::discriminant(c) == std::mem::discriminant(&ctx))); self.ctx.pop(); } @@ -166,7 +171,7 @@ impl Tokenizer<'_> { } } - fn check_path_since(&mut self, past: State) -> SyntaxKind { + fn check_path_since(&mut self, past: State, kind: SyntaxKind) -> SyntaxKind { self.consume(is_valid_path_char); if self.remaining().starts_with("${") { self.ctx.push(Context::InterpolStart); @@ -175,9 +180,9 @@ impl Tokenizer<'_> { } else if self.str_since(past).contains("//") { return TOKEN_ERROR; } else { - self.pop_ctx(Context::Path); + self.pop_ctx(Context::Path(kind)); } - TOKEN_PATH + kind } fn next_inner(&mut self) -> Option { @@ -195,14 +200,23 @@ impl Tokenizer<'_> { unreachable!() } } - Some(Context::Path) => { - if self.starts_with_bump("${") { + Some(Context::Path(path_kind)) => { + let path_kind = *path_kind; + + // Search paths () don't support interpolation + if path_kind == TOKEN_PATH_SEARCH { + if self.peek().map_or(false, is_valid_path_char) { + return Some(self.check_path_since(start, path_kind)); + } else { + self.pop_ctx(Context::Path(path_kind)); + } + } else if self.starts_with_bump("${") { self.ctx.push(Context::Interpol { brackets: 0 }); return Some(TOKEN_INTERPOL_START); } else if self.peek().map_or(false, is_valid_path_char) { - return Some(self.check_path_since(start)); + return Some(self.check_path_since(start, path_kind)); } else { - self.pop_ctx(Context::Path); + self.pop_ctx(Context::Path(path_kind)); } } Some(Context::StringBody { multiline }) => { @@ -267,8 +281,26 @@ impl Tokenizer<'_> { } // Check if it's a path - let store_path = self.peek() == Some('<'); - let kind = { + // First, simple prefix based detection for absolute ("/...") or home relative ("~/...") paths. + // For absolute paths, we need to be careful to not interfere with operators + let kind = if self.peek() == Some('/') { + // For absolute paths, we need to have a valid path character after the '/' + // AND we need to make sure it's not an operator like // + if self.remaining().starts_with("//") { + None // This could be an update operator, let the operator matcher handle it + } else { + let second_char = self.remaining().chars().nth(1); + if second_char.map_or(false, is_valid_path_char) { + Some(IdentType::PathAbs) + } else { + None // Not a path, might be division operator or something else + } + } + } else if self.peek() == Some('~') { + Some(IdentType::PathHome) + } else { + // Fallback to heuristic previously used for relative, search and URI paths. + let store_path = self.peek() == Some('<'); let skipped = self .remaining() .chars() @@ -280,27 +312,57 @@ impl Tokenizer<'_> { let mut lookahead = self.remaining().chars().skip(skipped.chars().count()); - match (lookahead.next(), lookahead.next()) { + let res = match (lookahead.next(), lookahead.next()) { // a//b parses as Update(a, b) (Some('/'), Some('/')) => None, (Some('/'), Some('*')) => None, - (Some('/'), Some(c)) if !c.is_whitespace() => Some(IdentType::Path), - (Some('>'), _) => Some(IdentType::Store), + (Some('/'), Some(c)) if !c.is_whitespace() => { + // If the first character of the yet‑to‑be‑consumed token was '/', we treat it as absolute later. + // Otherwise it's a relative path. + Some(IdentType::PathRel) + } + (Some('>'), _) => Some(IdentType::PathSearch), (Some(':'), Some(c)) if is_valid_uri_char(c) && !skipped.contains('_') => { Some(IdentType::Uri) } _ => None, - } + }; + res }; let c = self.next()?; - if c == '~' || kind == Some(IdentType::Path) { - return Some(if c == '~' && self.next() != Some('/') { - TOKEN_ERROR - } else { - self.push_ctx(Context::Path); - self.check_path_since(start) + // First, handle specific operators that might be confused with paths + if c == '/' && self.peek() == Some('/') { + // This is the update operator '//' + self.next().unwrap(); // Consume the second '/' + return Some(TOKEN_UPDATE); + } + + if c == '~' + || matches!(kind, Some(IdentType::PathAbs | IdentType::PathRel)) + || (c == '/' && self.peek().map_or(false, is_valid_path_char)) + { + return Some({ + // Determine the concrete path token kind to use + let token_kind = if c == '~' { + // we've just consumed '~', ensure '/' exists already consumed by next() + if self.next() != Some('/') { + return Some(TOKEN_ERROR); + } + TOKEN_PATH_HOME + } else { + // For abs vs rel: If the first char we consumed was '/', it's absolute. + if c == '/' { + TOKEN_PATH_ABS + } else { + TOKEN_PATH_REL + } + }; + + // Store the kind in context so subsequent segments are consistent + self.push_ctx(Context::Path(token_kind)); + self.check_path_since(start, token_kind) }); } @@ -370,12 +432,12 @@ impl Tokenizer<'_> { self.next().unwrap(); TOKEN_PIPE_LEFT } - '<' if kind == Some(IdentType::Store) => { + '<' if kind == Some(IdentType::PathSearch) => { self.consume(is_valid_path_char); if self.next() != Some('>') { TOKEN_ERROR } else { - TOKEN_PATH + TOKEN_PATH_SEARCH } } '&' if self.peek() == Some('&') => { @@ -409,7 +471,7 @@ impl Tokenizer<'_> { let kind = match kind { // It's detected as store if it ends with >, but if it // didn't start with <, that's wrong - Some(IdentType::Store) | None => IdentType::Ident, + Some(IdentType::PathSearch) | None => IdentType::Ident, Some(kind) => kind, }; self.consume(|c| match c { @@ -432,8 +494,10 @@ impl Tokenizer<'_> { _ => TOKEN_IDENT, }, IdentType::Uri => TOKEN_URI, - IdentType::Path => panic!("paths are checked earlier"), - IdentType::Store => panic!("store paths are checked earlier"), + IdentType::PathAbs | IdentType::PathRel | IdentType::PathHome => { + panic!("paths are checked earlier") + } + IdentType::PathSearch => panic!("search paths are checked earlier"), } } '"' => { diff --git a/test_data/parser/error/path_interp_no_separator.expect b/test_data/parser/error/path_interp_no_separator.expect index 87eb64e..ccf1b76 100644 --- a/test_data/parser/error/path_interp_no_separator.expect +++ b/test_data/parser/error/path_interp_no_separator.expect @@ -6,5 +6,5 @@ NODE_ROOT@0..7 TOKEN_INTERPOL_START@1..3 "${" TOKEN_IDENT@3..4 "b" TOKEN_INTERPOL_END@4..5 "}" - TOKEN_PATH@5..7 "/c" + TOKEN_PATH_ABS@5..7 "/c" diff --git a/test_data/parser/error/path_interp_trailing_slash.expect b/test_data/parser/error/path_interp_trailing_slash.expect index 7f62aaf..72e47ec 100644 --- a/test_data/parser/error/path_interp_trailing_slash.expect +++ b/test_data/parser/error/path_interp_trailing_slash.expect @@ -1,7 +1,7 @@ error: unexpected token at 8..9 NODE_ROOT@0..9 - NODE_PATH@0..8 - TOKEN_PATH@0..2 "./" + NODE_PATH_REL@0..8 + TOKEN_PATH_REL@0..2 "./" NODE_INTERPOL@2..8 TOKEN_INTERPOL_START@2..4 "${" NODE_IDENT@4..7 diff --git a/test_data/parser/error/path_store_interp.expect b/test_data/parser/error/path_store_interp.expect index 5ac24a7..142c36f 100644 --- a/test_data/parser/error/path_store_interp.expect +++ b/test_data/parser/error/path_store_interp.expect @@ -5,13 +5,13 @@ NODE_ROOT@0..20 NODE_APPLY@0..19 NODE_ERROR@0..1 TOKEN_LESS@0..1 "<" - NODE_PATH@1..19 - TOKEN_PATH@1..9 "nixpkgs/" + NODE_PATH_REL@1..19 + TOKEN_PATH_REL@1..9 "nixpkgs/" NODE_INTERPOL@9..15 TOKEN_INTERPOL_START@9..11 "${" NODE_IDENT@11..14 TOKEN_IDENT@11..14 "foo" TOKEN_INTERPOL_END@14..15 "}" - TOKEN_PATH@15..19 "/bar" + TOKEN_PATH_REL@15..19 "/bar" TOKEN_MORE@19..20 ">" diff --git a/test_data/parser/success/import_nixpkgs.expect b/test_data/parser/success/import_nixpkgs.expect index 28d65aa..63e65bf 100644 --- a/test_data/parser/success/import_nixpkgs.expect +++ b/test_data/parser/success/import_nixpkgs.expect @@ -3,8 +3,8 @@ NODE_ROOT@0..17 NODE_APPLY@0..15 NODE_IDENT@0..6 TOKEN_IDENT@0..6 "import" - NODE_PATH@6..15 - TOKEN_PATH@6..15 "" + NODE_PATH_SEARCH@6..15 + TOKEN_PATH_SEARCH@6..15 "" NODE_ATTR_SET@15..17 TOKEN_L_BRACE@15..16 "{" TOKEN_R_BRACE@16..17 "}" diff --git a/test_data/parser/success/path.expect b/test_data/parser/success/path.expect index 431231c..febb49d 100644 --- a/test_data/parser/success/path.expect +++ b/test_data/parser/success/path.expect @@ -9,8 +9,8 @@ NODE_ROOT@0..107 TOKEN_WHITESPACE@7..8 " " TOKEN_ASSIGN@8..9 "=" TOKEN_WHITESPACE@9..10 " " - NODE_PATH@10..20 - TOKEN_PATH@10..20 "/nix/store" + NODE_PATH_ABS@10..20 + TOKEN_PATH_ABS@10..20 "/nix/store" TOKEN_SEMICOLON@20..21 ";" TOKEN_WHITESPACE@21..24 "\n " NODE_ATTRPATH_VALUE@24..50 @@ -20,8 +20,8 @@ NODE_ROOT@0..107 TOKEN_WHITESPACE@28..29 " " TOKEN_ASSIGN@29..30 "=" TOKEN_WHITESPACE@30..31 " " - NODE_PATH@31..49 - TOKEN_PATH@31..49 "~/.nix-profile/bin" + NODE_PATH_HOME@31..49 + TOKEN_PATH_HOME@31..49 "~/.nix-profile/bin" TOKEN_SEMICOLON@49..50 ";" TOKEN_WHITESPACE@50..53 "\n " NODE_ATTRPATH_VALUE@53..79 @@ -31,8 +31,8 @@ NODE_ROOT@0..107 TOKEN_WHITESPACE@56..57 " " TOKEN_ASSIGN@57..58 "=" TOKEN_WHITESPACE@58..59 " " - NODE_PATH@59..78 - TOKEN_PATH@59..78 "./configuration.nix" + NODE_PATH_REL@59..78 + TOKEN_PATH_REL@59..78 "./configuration.nix" TOKEN_SEMICOLON@78..79 ";" TOKEN_WHITESPACE@79..82 "\n " NODE_ATTRPATH_VALUE@82..105 @@ -42,8 +42,8 @@ NODE_ROOT@0..107 TOKEN_WHITESPACE@87..88 " " TOKEN_ASSIGN@88..89 "=" TOKEN_WHITESPACE@89..90 " " - NODE_PATH@90..104 - TOKEN_PATH@90..104 "" + NODE_PATH_SEARCH@90..104 + TOKEN_PATH_SEARCH@90..104 "" TOKEN_SEMICOLON@104..105 ";" TOKEN_WHITESPACE@105..106 "\n" TOKEN_R_BRACE@106..107 "}" diff --git a/test_data/parser/success/path_interp.expect b/test_data/parser/success/path_interp.expect index 91c21ef..50c003b 100644 --- a/test_data/parser/success/path_interp.expect +++ b/test_data/parser/success/path_interp.expect @@ -15,14 +15,14 @@ NODE_ROOT@0..91 TOKEN_IDENT@10..11 "f" TOKEN_COLON@11..12 ":" TOKEN_WHITESPACE@12..13 " " - NODE_PATH@13..28 - TOKEN_PATH@13..18 "./foo" + NODE_PATH_REL@13..28 + TOKEN_PATH_REL@13..18 "./foo" NODE_INTERPOL@18..24 TOKEN_INTERPOL_START@18..20 "${" NODE_IDENT@20..23 TOKEN_IDENT@20..23 "bar" TOKEN_INTERPOL_END@23..24 "}" - TOKEN_PATH@24..28 "/baz" + TOKEN_PATH_REL@24..28 "/baz" TOKEN_SEMICOLON@28..29 ";" TOKEN_WHITESPACE@29..32 "\n " NODE_ATTRPATH_VALUE@32..86 @@ -39,40 +39,40 @@ NODE_ROOT@0..91 NODE_IDENT@36..37 TOKEN_IDENT@36..37 "a" TOKEN_WHITESPACE@37..38 " " - NODE_PATH@38..43 - TOKEN_PATH@38..43 "./bar" + NODE_PATH_REL@38..43 + TOKEN_PATH_REL@38..43 "./bar" TOKEN_WHITESPACE@43..44 " " - NODE_PATH@44..53 - TOKEN_PATH@44..49 "./baz" + NODE_PATH_REL@44..53 + TOKEN_PATH_REL@44..49 "./baz" NODE_INTERPOL@49..53 TOKEN_INTERPOL_START@49..51 "${" NODE_IDENT@51..52 TOKEN_IDENT@51..52 "x" TOKEN_INTERPOL_END@52..53 "}" TOKEN_WHITESPACE@53..54 " " - NODE_PATH@54..66 - TOKEN_PATH@54..61 "./snens" + NODE_PATH_REL@54..66 + TOKEN_PATH_REL@54..61 "./snens" NODE_INTERPOL@61..65 TOKEN_INTERPOL_START@61..63 "${" NODE_IDENT@63..64 TOKEN_IDENT@63..64 "x" TOKEN_INTERPOL_END@64..65 "}" - TOKEN_PATH@65..66 "y" + TOKEN_PATH_REL@65..66 "y" TOKEN_WHITESPACE@66..67 " " - NODE_PATH@67..85 - TOKEN_PATH@67..72 "./qux" + NODE_PATH_REL@67..85 + TOKEN_PATH_REL@67..72 "./qux" NODE_INTERPOL@72..76 TOKEN_INTERPOL_START@72..74 "${" NODE_IDENT@74..75 TOKEN_IDENT@74..75 "x" TOKEN_INTERPOL_END@75..76 "}" - TOKEN_PATH@76..77 "." + TOKEN_PATH_REL@76..77 "." NODE_INTERPOL@77..81 TOKEN_INTERPOL_START@77..79 "${" NODE_IDENT@79..80 TOKEN_IDENT@79..80 "y" TOKEN_INTERPOL_END@80..81 "}" - TOKEN_PATH@81..85 ".z/w" + TOKEN_PATH_REL@81..85 ".z/w" TOKEN_SEMICOLON@85..86 ";" TOKEN_WHITESPACE@86..87 "\n" TOKEN_IN@87..89 "in" diff --git a/test_data/parser/success/path_interp_no_prefix.expect b/test_data/parser/success/path_interp_no_prefix.expect index bf771f7..435a45c 100644 --- a/test_data/parser/success/path_interp_no_prefix.expect +++ b/test_data/parser/success/path_interp_no_prefix.expect @@ -1,6 +1,6 @@ NODE_ROOT@0..6 - NODE_PATH@0..6 - TOKEN_PATH@0..2 "a/" + NODE_PATH_REL@0..6 + TOKEN_PATH_REL@0..2 "a/" NODE_INTERPOL@2..6 TOKEN_INTERPOL_START@2..4 "${" NODE_IDENT@4..5 diff --git a/test_data/parser/success/path_no_newline.expect b/test_data/parser/success/path_no_newline.expect index a34efbc..f4336ab 100644 --- a/test_data/parser/success/path_no_newline.expect +++ b/test_data/parser/success/path_no_newline.expect @@ -3,7 +3,7 @@ NODE_ROOT@0..11 NODE_IDENT@0..6 TOKEN_IDENT@0..6 "import" TOKEN_WHITESPACE@6..7 " " - NODE_PATH@7..10 - TOKEN_PATH@7..10 "./." + NODE_PATH_REL@7..10 + TOKEN_PATH_REL@7..10 "./." TOKEN_WHITESPACE@10..11 "\n" diff --git a/test_data/parser/success/with-import-let-in.expect b/test_data/parser/success/with-import-let-in.expect index 74bbcc6..0d90fcc 100644 --- a/test_data/parser/success/with-import-let-in.expect +++ b/test_data/parser/success/with-import-let-in.expect @@ -6,8 +6,8 @@ NODE_ROOT@0..150 NODE_IDENT@5..11 TOKEN_IDENT@5..11 "import" TOKEN_WHITESPACE@11..12 " " - NODE_PATH@12..28 - TOKEN_PATH@12..28 "./simple-set.nix" + NODE_PATH_REL@12..28 + TOKEN_PATH_REL@12..28 "./simple-set.nix" TOKEN_SEMICOLON@28..29 ";" TOKEN_WHITESPACE@29..31 "\n\n" NODE_LET_IN@31..150 diff --git a/test_data/tokenizer/error/path_interp_trailing_slash.expect b/test_data/tokenizer/error/path_interp_trailing_slash.expect index 4ba40ce..8addf6f 100644 --- a/test_data/tokenizer/error/path_interp_trailing_slash.expect +++ b/test_data/tokenizer/error/path_interp_trailing_slash.expect @@ -1,4 +1,4 @@ -TOKEN_PATH, "./" +TOKEN_PATH_REL, "./" TOKEN_INTERPOL_START, "${" TOKEN_IDENT, "foo" TOKEN_INTERPOL_END, "}" diff --git a/test_data/tokenizer/success/path_absolute.expect b/test_data/tokenizer/success/path_absolute.expect index 5b0ea5d..ed29043 100644 --- a/test_data/tokenizer/success/path_absolute.expect +++ b/test_data/tokenizer/success/path_absolute.expect @@ -1 +1 @@ -TOKEN_PATH, "/hello/world" +TOKEN_PATH_ABS, "/hello/world" diff --git a/test_data/tokenizer/success/path_home.expect b/test_data/tokenizer/success/path_home.expect index 16e19ff..f7178f5 100644 --- a/test_data/tokenizer/success/path_home.expect +++ b/test_data/tokenizer/success/path_home.expect @@ -1 +1 @@ -TOKEN_PATH, "~/hello/world" +TOKEN_PATH_HOME, "~/hello/world" diff --git a/test_data/tokenizer/success/path_interp.expect b/test_data/tokenizer/success/path_interp.expect index 54fcaa1..12c83f0 100644 --- a/test_data/tokenizer/success/path_interp.expect +++ b/test_data/tokenizer/success/path_interp.expect @@ -1,4 +1,4 @@ -TOKEN_PATH, "./" +TOKEN_PATH_REL, "./" TOKEN_INTERPOL_START, "${" TOKEN_IDENT, "foo" TOKEN_INTERPOL_END, "}" diff --git a/test_data/tokenizer/success/path_interp_apply.expect b/test_data/tokenizer/success/path_interp_apply.expect index b7cfcae..8847b88 100644 --- a/test_data/tokenizer/success/path_interp_apply.expect +++ b/test_data/tokenizer/success/path_interp_apply.expect @@ -1,4 +1,4 @@ -TOKEN_PATH, "./" +TOKEN_PATH_REL, "./" TOKEN_INTERPOL_START, "${" TOKEN_IDENT, "foo" TOKEN_INTERPOL_END, "}" diff --git a/test_data/tokenizer/success/path_interp_multiple.expect b/test_data/tokenizer/success/path_interp_multiple.expect index fa7dbb4..c08e687 100644 --- a/test_data/tokenizer/success/path_interp_multiple.expect +++ b/test_data/tokenizer/success/path_interp_multiple.expect @@ -1,4 +1,4 @@ -TOKEN_PATH, "./" +TOKEN_PATH_REL, "./" TOKEN_INTERPOL_START, "${" TOKEN_IDENT, "foo" TOKEN_INTERPOL_END, "}" diff --git a/test_data/tokenizer/success/path_interp_multiple2.expect b/test_data/tokenizer/success/path_interp_multiple2.expect index dc3c8f5..822196a 100644 --- a/test_data/tokenizer/success/path_interp_multiple2.expect +++ b/test_data/tokenizer/success/path_interp_multiple2.expect @@ -1,8 +1,8 @@ -TOKEN_PATH, "./" +TOKEN_PATH_REL, "./" TOKEN_INTERPOL_START, "${" TOKEN_IDENT, "foo" TOKEN_INTERPOL_END, "}" -TOKEN_PATH, "a" +TOKEN_PATH_REL, "a" TOKEN_INTERPOL_START, "${" TOKEN_IDENT, "bar" TOKEN_INTERPOL_END, "}" diff --git a/test_data/tokenizer/success/path_interp_then_plain.expect b/test_data/tokenizer/success/path_interp_then_plain.expect index 3e5f0bb..5663841 100644 --- a/test_data/tokenizer/success/path_interp_then_plain.expect +++ b/test_data/tokenizer/success/path_interp_then_plain.expect @@ -1,5 +1,5 @@ -TOKEN_PATH, "./" +TOKEN_PATH_REL, "./" TOKEN_INTERPOL_START, "${" TOKEN_IDENT, "foo" TOKEN_INTERPOL_END, "}" -TOKEN_PATH, ".jpg" +TOKEN_PATH_REL, ".jpg" diff --git a/test_data/tokenizer/success/path_isnt_math.expect b/test_data/tokenizer/success/path_isnt_math.expect index eb98c91..eef6efa 100644 --- a/test_data/tokenizer/success/path_isnt_math.expect +++ b/test_data/tokenizer/success/path_isnt_math.expect @@ -1 +1 @@ -TOKEN_PATH, "a+3/5+b" +TOKEN_PATH_REL, "a+3/5+b" diff --git a/test_data/tokenizer/success/path_no_newline.expect b/test_data/tokenizer/success/path_no_newline.expect index f489e04..9ecc59e 100644 --- a/test_data/tokenizer/success/path_no_newline.expect +++ b/test_data/tokenizer/success/path_no_newline.expect @@ -1,5 +1,5 @@ TOKEN_IDENT, "import" TOKEN_WHITESPACE, " " -TOKEN_PATH, "./." +TOKEN_PATH_REL, "./." TOKEN_WHITESPACE, " " diff --git a/test_data/tokenizer/success/path_relative.expect b/test_data/tokenizer/success/path_relative.expect index 3420226..2bc5054 100644 --- a/test_data/tokenizer/success/path_relative.expect +++ b/test_data/tokenizer/success/path_relative.expect @@ -1 +1 @@ -TOKEN_PATH, "hello/world" +TOKEN_PATH_REL, "hello/world" diff --git a/test_data/tokenizer/success/path_relative_prefix.expect b/test_data/tokenizer/success/path_relative_prefix.expect index b06869c..e76c5e6 100644 --- a/test_data/tokenizer/success/path_relative_prefix.expect +++ b/test_data/tokenizer/success/path_relative_prefix.expect @@ -1 +1 @@ -TOKEN_PATH, "./hello/world" +TOKEN_PATH_REL, "./hello/world" diff --git a/test_data/tokenizer/success/path_store.expect b/test_data/tokenizer/success/path_store.expect index 4a4b090..c97778d 100644 --- a/test_data/tokenizer/success/path_store.expect +++ b/test_data/tokenizer/success/path_store.expect @@ -1 +1 @@ -TOKEN_PATH, "" +TOKEN_PATH_SEARCH, "" diff --git a/test_data/tokenizer/success/path_underscore.expect b/test_data/tokenizer/success/path_underscore.expect index 7845028..4770141 100644 --- a/test_data/tokenizer/success/path_underscore.expect +++ b/test_data/tokenizer/success/path_underscore.expect @@ -1 +1 @@ -TOKEN_PATH, "hello_/world" +TOKEN_PATH_REL, "hello_/world" From 9291a4de24671577da3efdc5ab7fe7eaf77922ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 24 Jun 2025 13:53:53 -0500 Subject: [PATCH 3/3] fix: improve path tokenization and add comprehensive tests - Fix path tokenization to correctly handle edge cases like ~/bar paths - Prevent //* from being tokenized as a path (it's division followed by multiplication) - Add validation for empty search paths <> - Add comprehensive test coverage for various path types including: - Absolute paths with special characters and nix store paths - Home paths with hidden directories - Search paths with complex names - Paths with numbers, plus signs, and other valid characters - Error cases for double slashes and invalid home paths - URI tokenization tests --- src/tokenizer.rs | 11 +- .../parser/error/path_multiple_errors.expect | 59 ++++++++ .../parser/error/path_multiple_errors.nix | 7 + .../parser/success/path_complex_mix.expect | 142 ++++++++++++++++++ test_data/parser/success/path_complex_mix.nix | 17 +++ test_data/parser/success/uri_various.expect | 96 ++++++++++++ test_data/parser/success/uri_various.nix | 12 ++ .../error/path_double_slash_middle.expect | 1 + .../error/path_double_slash_middle.nix | 1 + .../error/path_home_double_slash.expect | 1 + .../error/path_home_double_slash.nix | 1 + .../success/path_abs_nix_store.expect | 1 + .../tokenizer/success/path_abs_nix_store.nix | 1 + .../tokenizer/success/path_home_hidden.expect | 1 + .../tokenizer/success/path_home_hidden.nix | 1 + .../success/path_operators_mix.expect | 9 ++ .../tokenizer/success/path_operators_mix.nix | 1 + .../success/path_relative_complex.expect | 1 + .../success/path_relative_complex.nix | 1 + .../success/path_search_complex.expect | 1 + .../tokenizer/success/path_search_complex.nix | 1 + .../success/path_search_with_dash.expect | 1 + .../success/path_search_with_dash.nix | 1 + .../success/path_with_numbers.expect | 1 + .../tokenizer/success/path_with_numbers.nix | 1 + .../tokenizer/success/path_with_plus.expect | 1 + .../tokenizer/success/path_with_plus.nix | 1 + .../tokenizer/success/uri_git_ssh.expect | 1 + test_data/tokenizer/success/uri_git_ssh.nix | 1 + test_data/tokenizer/success/uri_https.expect | 1 + test_data/tokenizer/success/uri_https.nix | 1 + 31 files changed, 373 insertions(+), 3 deletions(-) create mode 100644 test_data/parser/error/path_multiple_errors.expect create mode 100644 test_data/parser/error/path_multiple_errors.nix create mode 100644 test_data/parser/success/path_complex_mix.expect create mode 100644 test_data/parser/success/path_complex_mix.nix create mode 100644 test_data/parser/success/uri_various.expect create mode 100644 test_data/parser/success/uri_various.nix create mode 100644 test_data/tokenizer/error/path_double_slash_middle.expect create mode 100644 test_data/tokenizer/error/path_double_slash_middle.nix create mode 100644 test_data/tokenizer/error/path_home_double_slash.expect create mode 100644 test_data/tokenizer/error/path_home_double_slash.nix create mode 100644 test_data/tokenizer/success/path_abs_nix_store.expect create mode 100644 test_data/tokenizer/success/path_abs_nix_store.nix create mode 100644 test_data/tokenizer/success/path_home_hidden.expect create mode 100644 test_data/tokenizer/success/path_home_hidden.nix create mode 100644 test_data/tokenizer/success/path_operators_mix.expect create mode 100644 test_data/tokenizer/success/path_operators_mix.nix create mode 100644 test_data/tokenizer/success/path_relative_complex.expect create mode 100644 test_data/tokenizer/success/path_relative_complex.nix create mode 100644 test_data/tokenizer/success/path_search_complex.expect create mode 100644 test_data/tokenizer/success/path_search_complex.nix create mode 100644 test_data/tokenizer/success/path_search_with_dash.expect create mode 100644 test_data/tokenizer/success/path_search_with_dash.nix create mode 100644 test_data/tokenizer/success/path_with_numbers.expect create mode 100644 test_data/tokenizer/success/path_with_numbers.nix create mode 100644 test_data/tokenizer/success/path_with_plus.expect create mode 100644 test_data/tokenizer/success/path_with_plus.nix create mode 100644 test_data/tokenizer/success/uri_git_ssh.expect create mode 100644 test_data/tokenizer/success/uri_git_ssh.nix create mode 100644 test_data/tokenizer/success/uri_https.expect create mode 100644 test_data/tokenizer/success/uri_https.nix diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 88c412d..f879146 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -290,13 +290,13 @@ impl Tokenizer<'_> { None // This could be an update operator, let the operator matcher handle it } else { let second_char = self.remaining().chars().nth(1); - if second_char.map_or(false, is_valid_path_char) { + if second_char.map_or(false, |c| is_valid_path_char(c) && c != '*') { Some(IdentType::PathAbs) } else { None // Not a path, might be division operator or something else } } - } else if self.peek() == Some('~') { + } else if self.peek() == Some('~') && self.remaining().chars().nth(1) == Some('/') { Some(IdentType::PathHome) } else { // Fallback to heuristic previously used for relative, search and URI paths. @@ -433,8 +433,13 @@ impl Tokenizer<'_> { TOKEN_PIPE_LEFT } '<' if kind == Some(IdentType::PathSearch) => { + let content_start = self.state.offset; self.consume(is_valid_path_char); - if self.next() != Some('>') { + // Check if we consumed any content between < and > + if self.state.offset == content_start { + // Empty search path <> is not valid + TOKEN_LESS + } else if self.next() != Some('>') { TOKEN_ERROR } else { TOKEN_PATH_SEARCH diff --git a/test_data/parser/error/path_multiple_errors.expect b/test_data/parser/error/path_multiple_errors.expect new file mode 100644 index 0000000..02f3633 --- /dev/null +++ b/test_data/parser/error/path_multiple_errors.expect @@ -0,0 +1,59 @@ +error: unexpected TOKEN_ERROR at 57..66, wanted any of [TOKEN_L_PAREN, TOKEN_REC, TOKEN_L_BRACE, TOKEN_L_BRACK, TOKEN_STRING_START, TOKEN_IDENT] +error: unexpected TOKEN_ERROR at 84..93, wanted any of [TOKEN_L_PAREN, TOKEN_REC, TOKEN_L_BRACE, TOKEN_L_BRACK, TOKEN_STRING_START, TOKEN_IDENT] +error: unexpected TOKEN_ERROR at 108..110, wanted any of [TOKEN_L_PAREN, TOKEN_REC, TOKEN_L_BRACE, TOKEN_L_BRACK, TOKEN_STRING_START, TOKEN_IDENT] +error: unexpected TOKEN_ERROR at 128..133, wanted any of [TOKEN_L_PAREN, TOKEN_REC, TOKEN_L_BRACE, TOKEN_L_BRACK, TOKEN_STRING_START, TOKEN_IDENT] +NODE_ROOT@0..141 + NODE_ATTR_SET@0..141 + TOKEN_L_BRACE@0..1 "{" + TOKEN_WHITESPACE@1..4 "\n " + TOKEN_COMMENT@4..38 "# Multiple path error ..." + TOKEN_WHITESPACE@38..41 "\n " + NODE_ATTRPATH_VALUE@41..67 + NODE_ATTRPATH@41..54 + NODE_IDENT@41..54 + TOKEN_IDENT@41..54 "trailingSlash" + TOKEN_WHITESPACE@54..55 " " + TOKEN_ASSIGN@55..56 "=" + TOKEN_WHITESPACE@56..57 " " + NODE_ERROR@57..66 + TOKEN_ERROR@57..66 "/foo/bar/" + TOKEN_SEMICOLON@66..67 ";" + TOKEN_WHITESPACE@67..70 "\n " + NODE_ATTRPATH_VALUE@70..94 + NODE_ATTRPATH@70..81 + NODE_IDENT@70..81 + TOKEN_IDENT@70..81 "doubleSlash" + TOKEN_WHITESPACE@81..82 " " + TOKEN_ASSIGN@82..83 "=" + TOKEN_WHITESPACE@83..84 " " + NODE_ERROR@84..93 + TOKEN_ERROR@84..93 "/foo//bar" + TOKEN_SEMICOLON@93..94 ";" + TOKEN_WHITESPACE@94..97 "\n " + NODE_ATTRPATH_VALUE@97..111 + NODE_ATTRPATH@97..105 + NODE_IDENT@97..105 + TOKEN_IDENT@97..105 "bareHome" + TOKEN_WHITESPACE@105..106 " " + TOKEN_ASSIGN@106..107 "=" + TOKEN_WHITESPACE@107..108 " " + NODE_ERROR@108..110 + TOKEN_ERROR@108..110 "~/" + TOKEN_SEMICOLON@110..111 ";" + TOKEN_WHITESPACE@111..114 "\n " + NODE_ATTRPATH_VALUE@114..139 + NODE_ATTRPATH@114..125 + NODE_IDENT@114..125 + TOKEN_IDENT@114..125 "tildeMiddle" + TOKEN_WHITESPACE@125..126 " " + TOKEN_ASSIGN@126..127 "=" + TOKEN_WHITESPACE@127..128 " " + NODE_APPLY@128..138 + NODE_ERROR@128..133 + TOKEN_ERROR@128..133 "/foo/" + NODE_PATH_HOME@133..138 + TOKEN_PATH_HOME@133..138 "~/bar" + TOKEN_SEMICOLON@138..139 ";" + TOKEN_WHITESPACE@139..140 "\n" + TOKEN_R_BRACE@140..141 "}" + diff --git a/test_data/parser/error/path_multiple_errors.nix b/test_data/parser/error/path_multiple_errors.nix new file mode 100644 index 0000000..69e7d5f --- /dev/null +++ b/test_data/parser/error/path_multiple_errors.nix @@ -0,0 +1,7 @@ +{ + # Multiple path errors in one file + trailingSlash = /foo/bar/; + doubleSlash = /foo//bar; + bareHome = ~/; + tildeMiddle = /foo/~/bar; +} \ No newline at end of file diff --git a/test_data/parser/success/path_complex_mix.expect b/test_data/parser/success/path_complex_mix.expect new file mode 100644 index 0000000..e4b0cdf --- /dev/null +++ b/test_data/parser/success/path_complex_mix.expect @@ -0,0 +1,142 @@ +NODE_ROOT@0..402 + NODE_ATTR_SET@0..402 + TOKEN_L_BRACE@0..1 "{" + TOKEN_WHITESPACE@1..4 "\n " + TOKEN_COMMENT@4..36 "# Various path types ..." + TOKEN_WHITESPACE@36..39 "\n " + NODE_ATTRPATH_VALUE@39..72 + NODE_ATTRPATH@39..42 + NODE_IDENT@39..42 + TOKEN_IDENT@39..42 "abs" + TOKEN_WHITESPACE@42..43 " " + TOKEN_ASSIGN@43..44 "=" + TOKEN_WHITESPACE@44..45 " " + NODE_PATH_ABS@45..71 + TOKEN_PATH_ABS@45..71 "/nix/store/hash-name- ..." + TOKEN_SEMICOLON@71..72 ";" + TOKEN_WHITESPACE@72..75 "\n " + NODE_ATTRPATH_VALUE@75..98 + NODE_ATTRPATH@75..78 + NODE_IDENT@75..78 + TOKEN_IDENT@75..78 "rel" + TOKEN_WHITESPACE@78..79 " " + TOKEN_ASSIGN@79..80 "=" + TOKEN_WHITESPACE@80..81 " " + NODE_PATH_REL@81..97 + TOKEN_PATH_REL@81..97 "./foo/bar/../baz" + TOKEN_SEMICOLON@97..98 ";" + TOKEN_WHITESPACE@98..101 "\n " + NODE_ATTRPATH_VALUE@101..135 + NODE_ATTRPATH@101..105 + NODE_IDENT@101..105 + TOKEN_IDENT@101..105 "home" + TOKEN_WHITESPACE@105..106 " " + TOKEN_ASSIGN@106..107 "=" + TOKEN_WHITESPACE@107..108 " " + NODE_PATH_HOME@108..134 + TOKEN_PATH_HOME@108..134 "~/.config/nixpkgs/ove ..." + TOKEN_SEMICOLON@134..135 ";" + TOKEN_WHITESPACE@135..138 "\n " + NODE_ATTRPATH_VALUE@138..169 + NODE_ATTRPATH@138..144 + NODE_IDENT@138..144 + TOKEN_IDENT@138..144 "search" + TOKEN_WHITESPACE@144..145 " " + TOKEN_ASSIGN@145..146 "=" + TOKEN_WHITESPACE@146..147 " " + NODE_PATH_SEARCH@147..168 + TOKEN_PATH_SEARCH@147..168 "" + TOKEN_SEMICOLON@168..169 ";" + TOKEN_WHITESPACE@169..175 "\n \n " + TOKEN_COMMENT@175..206 "# Paths with special ..." + TOKEN_WHITESPACE@206..209 "\n " + NODE_ATTRPATH_VALUE@209..234 + NODE_ATTRPATH@209..220 + NODE_IDENT@209..220 + TOKEN_IDENT@209..220 "withNumbers" + TOKEN_WHITESPACE@220..221 " " + TOKEN_ASSIGN@221..222 "=" + TOKEN_WHITESPACE@222..223 " " + NODE_PATH_ABS@223..233 + TOKEN_PATH_ABS@223..233 "/usr/lib64" + TOKEN_SEMICOLON@233..234 ";" + TOKEN_WHITESPACE@234..237 "\n " + NODE_ATTRPATH_VALUE@237..262 + NODE_ATTRPATH@237..245 + NODE_IDENT@237..245 + TOKEN_IDENT@237..245 "withDots" + TOKEN_WHITESPACE@245..246 " " + TOKEN_ASSIGN@246..247 "=" + TOKEN_WHITESPACE@247..248 " " + NODE_PATH_REL@248..261 + TOKEN_PATH_REL@248..261 "./foo.bar.baz" + TOKEN_SEMICOLON@261..262 ";" + TOKEN_WHITESPACE@262..265 "\n " + NODE_ATTRPATH_VALUE@265..286 + NODE_ATTRPATH@265..273 + NODE_IDENT@265..273 + TOKEN_IDENT@265..273 "withPlus" + TOKEN_WHITESPACE@273..274 " " + TOKEN_ASSIGN@274..275 "=" + TOKEN_WHITESPACE@275..276 " " + NODE_PATH_REL@276..285 + TOKEN_PATH_REL@276..285 "./foo+bar" + TOKEN_SEMICOLON@285..286 ";" + TOKEN_WHITESPACE@286..289 "\n " + NODE_ATTRPATH_VALUE@289..319 + NODE_ATTRPATH@289..297 + NODE_IDENT@289..297 + TOKEN_IDENT@289..297 "withDash" + TOKEN_WHITESPACE@297..298 " " + TOKEN_ASSIGN@298..299 "=" + TOKEN_WHITESPACE@299..300 " " + NODE_PATH_ABS@300..318 + TOKEN_PATH_ABS@300..318 "/nix-store/package" + TOKEN_SEMICOLON@318..319 ";" + TOKEN_WHITESPACE@319..325 "\n \n " + TOKEN_COMMENT@325..342 "# Path operations" + TOKEN_WHITESPACE@342..345 "\n " + NODE_ATTRPATH_VALUE@345..369 + NODE_ATTRPATH@345..353 + NODE_IDENT@345..353 + TOKEN_IDENT@345..353 "combined" + TOKEN_WHITESPACE@353..354 " " + TOKEN_ASSIGN@354..355 "=" + TOKEN_WHITESPACE@355..356 " " + NODE_BIN_OP@356..368 + NODE_PATH_ABS@356..360 + TOKEN_PATH_ABS@356..360 "/foo" + TOKEN_WHITESPACE@360..361 " " + TOKEN_ADD@361..362 "+" + TOKEN_WHITESPACE@362..363 " " + NODE_PATH_REL@363..368 + TOKEN_PATH_REL@363..368 "./bar" + TOKEN_SEMICOLON@368..369 ";" + TOKEN_WHITESPACE@369..372 "\n " + NODE_ATTRPATH_VALUE@372..400 + NODE_ATTRPATH@372..378 + NODE_IDENT@372..378 + TOKEN_IDENT@372..378 "inList" + TOKEN_WHITESPACE@378..379 " " + TOKEN_ASSIGN@379..380 "=" + TOKEN_WHITESPACE@380..381 " " + NODE_LIST@381..399 + TOKEN_L_BRACK@381..382 "[" + TOKEN_WHITESPACE@382..383 " " + NODE_PATH_ABS@383..385 + TOKEN_PATH_ABS@383..385 "/a" + TOKEN_WHITESPACE@385..386 " " + NODE_PATH_REL@386..389 + TOKEN_PATH_REL@386..389 "./b" + TOKEN_WHITESPACE@389..390 " " + NODE_PATH_HOME@390..393 + TOKEN_PATH_HOME@390..393 "~/c" + TOKEN_WHITESPACE@393..394 " " + NODE_PATH_SEARCH@394..397 + TOKEN_PATH_SEARCH@394..397 "" + TOKEN_WHITESPACE@397..398 " " + TOKEN_R_BRACK@398..399 "]" + TOKEN_SEMICOLON@399..400 ";" + TOKEN_WHITESPACE@400..401 "\n" + TOKEN_R_BRACE@401..402 "}" + diff --git a/test_data/parser/success/path_complex_mix.nix b/test_data/parser/success/path_complex_mix.nix new file mode 100644 index 0000000..1397717 --- /dev/null +++ b/test_data/parser/success/path_complex_mix.nix @@ -0,0 +1,17 @@ +{ + # Various path types in one file + abs = /nix/store/hash-name-1.0.0; + rel = ./foo/bar/../baz; + home = ~/.config/nixpkgs/overlays; + search = ; + + # Paths with special characters + withNumbers = /usr/lib64; + withDots = ./foo.bar.baz; + withPlus = ./foo+bar; + withDash = /nix-store/package; + + # Path operations + combined = /foo + ./bar; + inList = [ /a ./b ~/c ]; +} \ No newline at end of file diff --git a/test_data/parser/success/uri_various.expect b/test_data/parser/success/uri_various.expect new file mode 100644 index 0000000..525c93f --- /dev/null +++ b/test_data/parser/success/uri_various.expect @@ -0,0 +1,96 @@ +error: unexpected TOKEN_ASSIGN at 361..388, wanted any of [TOKEN_SEMICOLON] +NODE_ROOT@0..391 + NODE_ATTR_SET@0..391 + TOKEN_L_BRACE@0..1 "{" + TOKEN_WHITESPACE@1..4 "\n " + NODE_ATTRPATH_VALUE@4..30 + NODE_ATTRPATH@4..8 + NODE_IDENT@4..8 + TOKEN_IDENT@4..8 "http" + TOKEN_WHITESPACE@8..9 " " + TOKEN_ASSIGN@9..10 "=" + TOKEN_WHITESPACE@10..11 " " + NODE_LITERAL@11..29 + TOKEN_URI@11..29 "http://example.com" + TOKEN_SEMICOLON@29..30 ";" + TOKEN_WHITESPACE@30..33 "\n " + NODE_ATTRPATH_VALUE@33..74 + NODE_ATTRPATH@33..38 + NODE_IDENT@33..38 + TOKEN_IDENT@33..38 "https" + TOKEN_WHITESPACE@38..39 " " + TOKEN_ASSIGN@39..40 "=" + TOKEN_WHITESPACE@40..41 " " + NODE_LITERAL@41..73 + TOKEN_URI@41..73 "https://github.com/Ni ..." + TOKEN_SEMICOLON@73..74 ";" + TOKEN_WHITESPACE@74..77 "\n " + NODE_ATTRPATH_VALUE@77..123 + NODE_ATTRPATH@77..80 + NODE_IDENT@77..80 + TOKEN_IDENT@77..80 "git" + TOKEN_WHITESPACE@80..81 " " + TOKEN_ASSIGN@81..82 "=" + TOKEN_WHITESPACE@82..83 " " + NODE_LITERAL@83..122 + TOKEN_URI@83..122 "git+ssh://git@github. ..." + TOKEN_SEMICOLON@122..123 ";" + TOKEN_WHITESPACE@123..126 "\n " + NODE_ATTRPATH_VALUE@126..164 + NODE_ATTRPATH@126..130 + NODE_IDENT@126..130 + TOKEN_IDENT@126..130 "file" + TOKEN_WHITESPACE@130..131 " " + TOKEN_ASSIGN@131..132 "=" + TOKEN_WHITESPACE@132..133 " " + NODE_LITERAL@133..163 + TOKEN_URI@133..163 "file:///home/user/doc ..." + TOKEN_SEMICOLON@163..164 ";" + TOKEN_WHITESPACE@164..167 "\n " + NODE_ATTRPATH_VALUE@167..211 + NODE_ATTRPATH@167..170 + NODE_IDENT@167..170 + TOKEN_IDENT@167..170 "ftp" + TOKEN_WHITESPACE@170..171 " " + TOKEN_ASSIGN@171..172 "=" + TOKEN_WHITESPACE@172..173 " " + NODE_LITERAL@173..210 + TOKEN_URI@173..210 "ftp://ftp.example.com ..." + TOKEN_SEMICOLON@210..211 ";" + TOKEN_WHITESPACE@211..217 "\n \n " + TOKEN_COMMENT@217..247 "# URIs with special c ..." + TOKEN_WHITESPACE@247..250 "\n " + NODE_ATTRPATH_VALUE@250..298 + NODE_ATTRPATH@250..259 + NODE_IDENT@250..259 + TOKEN_IDENT@250..259 "withQuery" + TOKEN_WHITESPACE@259..260 " " + TOKEN_ASSIGN@260..261 "=" + TOKEN_WHITESPACE@261..262 " " + NODE_LITERAL@262..297 + TOKEN_URI@262..297 "https://example.com?f ..." + TOKEN_SEMICOLON@297..298 ";" + TOKEN_WHITESPACE@298..301 "\n " + NODE_ATTRPATH_VALUE@301..389 + NODE_ATTRPATH@301..313 + NODE_IDENT@301..313 + TOKEN_IDENT@301..313 "withFragment" + TOKEN_WHITESPACE@313..314 " " + TOKEN_ASSIGN@314..315 "=" + TOKEN_WHITESPACE@315..316 " " + NODE_APPLY@316..360 + NODE_LITERAL@316..340 + TOKEN_URI@316..340 "https://example.com/page" + TOKEN_COMMENT@340..349 "#section;" + TOKEN_WHITESPACE@349..352 "\n " + NODE_IDENT@352..360 + TOKEN_IDENT@352..360 "withPort" + TOKEN_WHITESPACE@360..361 " " + NODE_ERROR@361..388 + TOKEN_ASSIGN@361..362 "=" + TOKEN_WHITESPACE@362..363 " " + TOKEN_URI@363..388 "http://localhost:8080 ..." + TOKEN_SEMICOLON@388..389 ";" + TOKEN_WHITESPACE@389..390 "\n" + TOKEN_R_BRACE@390..391 "}" + diff --git a/test_data/parser/success/uri_various.nix b/test_data/parser/success/uri_various.nix new file mode 100644 index 0000000..345b1a7 --- /dev/null +++ b/test_data/parser/success/uri_various.nix @@ -0,0 +1,12 @@ +{ + http = http://example.com; + https = https://github.com/NixOS/nixpkgs; + git = git+ssh://git@github.com/owner/repo.git; + file = file:///home/user/document.pdf; + ftp = ftp://ftp.example.com/pub/file.tar.gz; + + # URIs with special characters + withQuery = https://example.com?foo=bar&baz=qux; + withFragment = https://example.com/page#section; + withPort = http://localhost:8080/api; +} \ No newline at end of file diff --git a/test_data/tokenizer/error/path_double_slash_middle.expect b/test_data/tokenizer/error/path_double_slash_middle.expect new file mode 100644 index 0000000..fc92f86 --- /dev/null +++ b/test_data/tokenizer/error/path_double_slash_middle.expect @@ -0,0 +1 @@ +TOKEN_ERROR, "/foo//bar/baz" diff --git a/test_data/tokenizer/error/path_double_slash_middle.nix b/test_data/tokenizer/error/path_double_slash_middle.nix new file mode 100644 index 0000000..21b7bc9 --- /dev/null +++ b/test_data/tokenizer/error/path_double_slash_middle.nix @@ -0,0 +1 @@ +/foo//bar/baz \ No newline at end of file diff --git a/test_data/tokenizer/error/path_home_double_slash.expect b/test_data/tokenizer/error/path_home_double_slash.expect new file mode 100644 index 0000000..7ab7c30 --- /dev/null +++ b/test_data/tokenizer/error/path_home_double_slash.expect @@ -0,0 +1 @@ +TOKEN_ERROR, "~//foo" diff --git a/test_data/tokenizer/error/path_home_double_slash.nix b/test_data/tokenizer/error/path_home_double_slash.nix new file mode 100644 index 0000000..2075c87 --- /dev/null +++ b/test_data/tokenizer/error/path_home_double_slash.nix @@ -0,0 +1 @@ +~//foo \ No newline at end of file diff --git a/test_data/tokenizer/success/path_abs_nix_store.expect b/test_data/tokenizer/success/path_abs_nix_store.expect new file mode 100644 index 0000000..7c26b5c --- /dev/null +++ b/test_data/tokenizer/success/path_abs_nix_store.expect @@ -0,0 +1 @@ +TOKEN_PATH_ABS, "/nix/store/abc123def456-package-1.0.0/bin/executable" diff --git a/test_data/tokenizer/success/path_abs_nix_store.nix b/test_data/tokenizer/success/path_abs_nix_store.nix new file mode 100644 index 0000000..e2a8626 --- /dev/null +++ b/test_data/tokenizer/success/path_abs_nix_store.nix @@ -0,0 +1 @@ +/nix/store/abc123def456-package-1.0.0/bin/executable \ No newline at end of file diff --git a/test_data/tokenizer/success/path_home_hidden.expect b/test_data/tokenizer/success/path_home_hidden.expect new file mode 100644 index 0000000..8e9f93c --- /dev/null +++ b/test_data/tokenizer/success/path_home_hidden.expect @@ -0,0 +1 @@ +TOKEN_PATH_HOME, "~/.config/nixpkgs/config.nix" diff --git a/test_data/tokenizer/success/path_home_hidden.nix b/test_data/tokenizer/success/path_home_hidden.nix new file mode 100644 index 0000000..2dbe3ad --- /dev/null +++ b/test_data/tokenizer/success/path_home_hidden.nix @@ -0,0 +1 @@ +~/.config/nixpkgs/config.nix \ No newline at end of file diff --git a/test_data/tokenizer/success/path_operators_mix.expect b/test_data/tokenizer/success/path_operators_mix.expect new file mode 100644 index 0000000..8d239ee --- /dev/null +++ b/test_data/tokenizer/success/path_operators_mix.expect @@ -0,0 +1,9 @@ +TOKEN_PATH_ABS, "/foo" +TOKEN_WHITESPACE, " " +TOKEN_ADD, "+" +TOKEN_WHITESPACE, " " +TOKEN_PATH_ABS, "/bar" +TOKEN_WHITESPACE, " " +TOKEN_SUB, "-" +TOKEN_WHITESPACE, " " +TOKEN_PATH_REL, "./baz" diff --git a/test_data/tokenizer/success/path_operators_mix.nix b/test_data/tokenizer/success/path_operators_mix.nix new file mode 100644 index 0000000..7984b47 --- /dev/null +++ b/test_data/tokenizer/success/path_operators_mix.nix @@ -0,0 +1 @@ +/foo + /bar - ./baz \ No newline at end of file diff --git a/test_data/tokenizer/success/path_relative_complex.expect b/test_data/tokenizer/success/path_relative_complex.expect new file mode 100644 index 0000000..7b48464 --- /dev/null +++ b/test_data/tokenizer/success/path_relative_complex.expect @@ -0,0 +1 @@ +TOKEN_PATH_REL, "./a/b/../c/./d/e" diff --git a/test_data/tokenizer/success/path_relative_complex.nix b/test_data/tokenizer/success/path_relative_complex.nix new file mode 100644 index 0000000..eb0bf27 --- /dev/null +++ b/test_data/tokenizer/success/path_relative_complex.nix @@ -0,0 +1 @@ +./a/b/../c/./d/e \ No newline at end of file diff --git a/test_data/tokenizer/success/path_search_complex.expect b/test_data/tokenizer/success/path_search_complex.expect new file mode 100644 index 0000000..cdd1571 --- /dev/null +++ b/test_data/tokenizer/success/path_search_complex.expect @@ -0,0 +1 @@ +TOKEN_PATH_SEARCH, "" diff --git a/test_data/tokenizer/success/path_search_complex.nix b/test_data/tokenizer/success/path_search_complex.nix new file mode 100644 index 0000000..4cff468 --- /dev/null +++ b/test_data/tokenizer/success/path_search_complex.nix @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test_data/tokenizer/success/path_search_with_dash.expect b/test_data/tokenizer/success/path_search_with_dash.expect new file mode 100644 index 0000000..884f387 --- /dev/null +++ b/test_data/tokenizer/success/path_search_with_dash.expect @@ -0,0 +1 @@ +TOKEN_PATH_SEARCH, "" diff --git a/test_data/tokenizer/success/path_search_with_dash.nix b/test_data/tokenizer/success/path_search_with_dash.nix new file mode 100644 index 0000000..38f0c4a --- /dev/null +++ b/test_data/tokenizer/success/path_search_with_dash.nix @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test_data/tokenizer/success/path_with_numbers.expect b/test_data/tokenizer/success/path_with_numbers.expect new file mode 100644 index 0000000..f151c90 --- /dev/null +++ b/test_data/tokenizer/success/path_with_numbers.expect @@ -0,0 +1 @@ +TOKEN_PATH_ABS, "/usr/lib64/libc.so.6" diff --git a/test_data/tokenizer/success/path_with_numbers.nix b/test_data/tokenizer/success/path_with_numbers.nix new file mode 100644 index 0000000..cbe74da --- /dev/null +++ b/test_data/tokenizer/success/path_with_numbers.nix @@ -0,0 +1 @@ +/usr/lib64/libc.so.6 \ No newline at end of file diff --git a/test_data/tokenizer/success/path_with_plus.expect b/test_data/tokenizer/success/path_with_plus.expect new file mode 100644 index 0000000..43f5e47 --- /dev/null +++ b/test_data/tokenizer/success/path_with_plus.expect @@ -0,0 +1 @@ +TOKEN_PATH_REL, "./foo+bar/baz" diff --git a/test_data/tokenizer/success/path_with_plus.nix b/test_data/tokenizer/success/path_with_plus.nix new file mode 100644 index 0000000..9fff018 --- /dev/null +++ b/test_data/tokenizer/success/path_with_plus.nix @@ -0,0 +1 @@ +./foo+bar/baz \ No newline at end of file diff --git a/test_data/tokenizer/success/uri_git_ssh.expect b/test_data/tokenizer/success/uri_git_ssh.expect new file mode 100644 index 0000000..1369d75 --- /dev/null +++ b/test_data/tokenizer/success/uri_git_ssh.expect @@ -0,0 +1 @@ +TOKEN_URI, "git+ssh://git@github.com/NixOS/nix.git" diff --git a/test_data/tokenizer/success/uri_git_ssh.nix b/test_data/tokenizer/success/uri_git_ssh.nix new file mode 100644 index 0000000..24390ba --- /dev/null +++ b/test_data/tokenizer/success/uri_git_ssh.nix @@ -0,0 +1 @@ +git+ssh://git@github.com/NixOS/nix.git \ No newline at end of file diff --git a/test_data/tokenizer/success/uri_https.expect b/test_data/tokenizer/success/uri_https.expect new file mode 100644 index 0000000..8f27b7f --- /dev/null +++ b/test_data/tokenizer/success/uri_https.expect @@ -0,0 +1 @@ +TOKEN_URI, "https://github.com/NixOS/nixpkgs/archive/master.tar.gz" diff --git a/test_data/tokenizer/success/uri_https.nix b/test_data/tokenizer/success/uri_https.nix new file mode 100644 index 0000000..f8e54cc --- /dev/null +++ b/test_data/tokenizer/success/uri_https.nix @@ -0,0 +1 @@ +https://github.com/NixOS/nixpkgs/archive/master.tar.gz \ No newline at end of file