|
|
@ -78,12 +78,26 @@ impl Token {
|
|
|
|
fn same(&self, other: &Self) -> bool {
|
|
|
|
fn same(&self, other: &Self) -> bool {
|
|
|
|
use Token::*;
|
|
|
|
use Token::*;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
println!(
|
|
|
|
|
|
|
|
"check same self: {self:?} other: {other:?}",
|
|
|
|
|
|
|
|
self = self.text(),
|
|
|
|
|
|
|
|
other = other.text()
|
|
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
match (self, other) {
|
|
|
|
match (self, other) {
|
|
|
|
(Word(a), Word(b)) => a.text() == b.text(),
|
|
|
|
(Word(a), Word(b)) => a.text() == b.text(),
|
|
|
|
(Glob(a), Glob(b)) => a.text() == b.text(),
|
|
|
|
(Glob(a), Glob(b)) => a.text() == b.text(),
|
|
|
|
_ => false,
|
|
|
|
_ => false,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn text(&self) -> String {
|
|
|
|
|
|
|
|
use Token::*;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
match self {
|
|
|
|
|
|
|
|
Word(lexeme) | Glob(lexeme) => lexeme.text(),
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Tokenizer splits some input [Glyphs] into [Token] values
|
|
|
|
/// Tokenizer splits some input [Glyphs] into [Token] values
|
|
|
@ -107,10 +121,10 @@ impl<'text> Tokenizer<'text> {
|
|
|
|
match next.glyph {
|
|
|
|
match next.glyph {
|
|
|
|
_ if next.is_word() => Some(self.lex_bare_string(vec![next])),
|
|
|
|
_ if next.is_word() => Some(self.lex_bare_string(vec![next])),
|
|
|
|
_ if next.is_glob() => Some(self.lex_glob(vec![next])),
|
|
|
|
_ if next.is_glob() => Some(self.lex_glob(vec![next])),
|
|
|
|
'\\' => match self.source.pop() {
|
|
|
|
// '\\' => match self.source.pop() {
|
|
|
|
Ok(escaped) => Some(self.lex_bare_string(vec![escaped])),
|
|
|
|
// Ok(escaped) => Some(self.lex_bare_string(vec![escaped])),
|
|
|
|
Err(e) => Some(Err(e)),
|
|
|
|
// Err(e) => Some(Err(e)),
|
|
|
|
},
|
|
|
|
// },
|
|
|
|
'@' => Some(self.lex_var(vec![next])),
|
|
|
|
'@' => Some(self.lex_var(vec![next])),
|
|
|
|
'\'' => Some(self.lex_raw_string(vec![next])),
|
|
|
|
'\'' => Some(self.lex_raw_string(vec![next])),
|
|
|
|
'"' => Some(self.lex_interp_string(vec![next])),
|
|
|
|
'"' => Some(self.lex_interp_string(vec![next])),
|
|
|
@ -127,10 +141,10 @@ impl<'text> Tokenizer<'text> {
|
|
|
|
progress.push(self.source.pop()?);
|
|
|
|
progress.push(self.source.pop()?);
|
|
|
|
return self.lex_glob(progress);
|
|
|
|
return self.lex_glob(progress);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
'\\' => {
|
|
|
|
// '\\' => {
|
|
|
|
self.source.pop()?;
|
|
|
|
// self.source.pop()?;
|
|
|
|
progress.push(self.source.pop()?);
|
|
|
|
// progress.push(self.source.pop()?);
|
|
|
|
}
|
|
|
|
// }
|
|
|
|
_ => return Err(LexError::UnexpectedCharacter(self.source.pop()?)),
|
|
|
|
_ => return Err(LexError::UnexpectedCharacter(self.source.pop()?)),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -147,10 +161,10 @@ impl<'text> Tokenizer<'text> {
|
|
|
|
match next.glyph {
|
|
|
|
match next.glyph {
|
|
|
|
_ if next.glyph.is_whitespace() => break,
|
|
|
|
_ if next.glyph.is_whitespace() => break,
|
|
|
|
_ if next.is_glob() => progress.push(self.source.pop()?),
|
|
|
|
_ if next.is_glob() => progress.push(self.source.pop()?),
|
|
|
|
'\\' => {
|
|
|
|
// '\\' => {
|
|
|
|
self.source.pop()?;
|
|
|
|
// self.source.pop()?;
|
|
|
|
progress.push(self.source.pop()?);
|
|
|
|
// progress.push(self.source.pop()?);
|
|
|
|
}
|
|
|
|
// }
|
|
|
|
_ => return Err(LexError::UnexpectedCharacter(self.source.pop()?)),
|
|
|
|
_ => return Err(LexError::UnexpectedCharacter(self.source.pop()?)),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -271,6 +285,7 @@ mod tests {
|
|
|
|
assert_eq!(expected.len(), lexed.len());
|
|
|
|
assert_eq!(expected.len(), lexed.len());
|
|
|
|
|
|
|
|
|
|
|
|
for pair in zip(expected, lexed) {
|
|
|
|
for pair in zip(expected, lexed) {
|
|
|
|
|
|
|
|
println!("pair: {pair:?}");
|
|
|
|
assert!(pair.0.same(&pair.1));
|
|
|
|
assert!(pair.0.same(&pair.1));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -295,13 +310,6 @@ mod tests {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
reject! {
|
|
|
|
reject! {
|
|
|
|
// A slash on its own makes no sense
|
|
|
|
|
|
|
|
lonely_slash: r"\";
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// A slash is an escape character, so starting the escape sequence and then ending the
|
|
|
|
|
|
|
|
// input makes no sense
|
|
|
|
|
|
|
|
trailing_slash: r"one two three \";
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Vars aren't done yet
|
|
|
|
// Vars aren't done yet
|
|
|
|
var: "@name";
|
|
|
|
var: "@name";
|
|
|
|
|
|
|
|
|
|
|
@ -332,5 +340,7 @@ mod tests {
|
|
|
|
glob_4 "*x" [ glob("*x") ]
|
|
|
|
glob_4 "*x" [ glob("*x") ]
|
|
|
|
glob_5 "*.py" [ glob("*.py") ]
|
|
|
|
glob_5 "*.py" [ glob("*.py") ]
|
|
|
|
mixed_1 "ls *.py" [ word("ls") glob("*.py") ]
|
|
|
|
mixed_1 "ls *.py" [ word("ls") glob("*.py") ]
|
|
|
|
|
|
|
|
abs "cd c:\\one\\two" [ word("cd") word("c:\\one\\two")]
|
|
|
|
|
|
|
|
home "cd ~\\stuff" [ word("cd") word("~\\stuff") ]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|