summaryrefslogtreecommitdiff
path: root/src/lexer.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/lexer.rs')
-rw-r--r--src/lexer.rs105
1 files changed, 93 insertions, 12 deletions
diff --git a/src/lexer.rs b/src/lexer.rs
index f4e23f3..beb3b83 100644
--- a/src/lexer.rs
+++ b/src/lexer.rs
@@ -1,7 +1,5 @@
use nom;
-use std::str::FromStr;
-
use core::Const;
#[derive(Debug, PartialEq, Eq)]
@@ -51,6 +49,7 @@ pub enum Tok {
Bool(bool),
Integer(isize),
Natural(usize),
+ Text(String),
// Symbols
BraceL,
@@ -90,19 +89,20 @@ fn is_identifier_first_char(c: char) -> bool {
}
fn is_identifier_rest_char(c: char) -> bool {
- is_identifier_first_char(c) || is_decimal(c) || c == '/'
+ is_identifier_first_char(c) || c.is_digit(10) || c == '/'
}
-fn is_decimal(c: char) -> bool {
- c.is_digit(10)
+macro_rules! digits {
+ ($i:expr, $t:tt, $radix:expr) => {{
+ let r: nom::IResult<&str, $t> =
+ map_res!($i, take_while1_s!(call!(|c: char| c.is_digit($radix))),
+ |s| $t::from_str_radix(s, $radix));
+ r
+ }}
}
-named!(identifier<&str, &str>, recognize!(preceded!(
- take_while1_s!(is_identifier_first_char),
- take_while_s!(is_identifier_rest_char))
-));
-named!(natural<&str, &str>, preceded!(tag!("+"), take_while1_s!(is_decimal)));
-named!(integral<&str, isize>, map_res!(take_while1_s!(is_decimal), |s| isize::from_str(s)));
+named!(natural<&str, usize>, preceded!(tag!("+"), digits!(usize, 10)));
+named!(integral<&str, isize>, digits!(isize, 10));
named!(integer<&str, isize>, alt!(
preceded!(tag!("-"), map!(integral, |i: isize| -i)) |
integral
@@ -112,6 +112,11 @@ named!(boolean<&str, bool>, alt!(
value!(false, tag!("False"))
));
+named!(identifier<&str, &str>, recognize!(preceded!(
+ take_while1_s!(is_identifier_first_char),
+ take_while_s!(is_identifier_rest_char))
+));
+
/// Parse an identifier, ensuring a whole identifier is parsed and not just a prefix.
macro_rules! ident_tag {
($i:expr, $tag:expr) => {
@@ -128,6 +133,75 @@ macro_rules! ident_tag {
}
}
+fn string_escape_single(c: char) -> Option<&'static str> {
+ match c {
+ 'n' => Some("\n"),
+ 'r' => Some("\r"),
+ 't' => Some("\t"),
+ '"' => Some("\""),
+ '\'' => Some("'"),
+ '\\' => Some("\\"),
+ '0' => Some("\0"),
+ 'a' => Some("\x07"),
+ 'b' => Some("\x08"),
+ 'f' => Some("\x0c"),
+ 'v' => Some("\x0b"),
+ '&' => Some(""),
+ _ => None,
+ }
+}
+
+named!(string_escape_numeric<&str, char>, map_opt!(alt!(
+ preceded!(tag!("x"), digits!(u32, 16)) |
+ preceded!(tag!("o"), digits!(u32, 8)) |
+ digits!(u32, 10)
+), ::std::char::from_u32));
+
+fn string_lit_inner(input: &str) -> nom::IResult<&str, String> {
+ use nom::IResult::*;;
+ use nom::ErrorKind;
+ let mut s = String::new();
+ let mut cs = input.char_indices().peekable();
+ while let Some((i, c)) = cs.next() {
+ match c {
+ '"' => return nom::IResult::Done(&input[i..], s),
+ '\\' => match cs.next() {
+ Some((_, s)) if s.is_whitespace() => {
+ while cs.peek().map(|&(_, s)| s.is_whitespace()) == Some(true) {
+ let _ = cs.next();
+ }
+ if cs.next().map(|p| p.1) != Some('\\') {
+ return Error(error_position!(ErrorKind::Custom(4 /* FIXME */), input));
+ }
+ }
+ Some((j, ec)) => {
+ if let Some(esc) = string_escape_single(ec) {
+ s.push_str(esc);
+ // FIXME Named ASCII escapes and control character escapes
+ } else {
+ match string_escape_numeric(&input[j..]) {
+ Done(rest, esc) => {
+ let &(k, _) = cs.peek().unwrap();
+ // digits are always single byte ASCII characters
+ let consumed = input[k..].len() - rest.len();
+ for _ in 0..consumed { let _ = cs.next(); }
+ s.push(esc);
+ }
+ Incomplete(s) => return Incomplete(s),
+ Error(e) => return Error(e),
+ }
+ }
+ },
+ _ => return Error(error_position!(ErrorKind::Custom(5 /* FIXME */), input)),
+ },
+ _ => s.push(c),
+ }
+ }
+ Error(error_position!(ErrorKind::Custom(3 /* FIXME */), input))
+}
+
+named!(string_lit<&str, String>, delimited!(tag!("\""), string_lit_inner, tag!("\"")));
+
named!(keyword<&str, Keyword>, alt!(
value!(Keyword::Let, ident_tag!("let")) |
value!(Keyword::In, ident_tag!("in")) |
@@ -182,9 +256,10 @@ named!(token<&str, Tok>, alt!(
map!(keyword, Tok::Keyword) |
map!(builtin, Tok::Builtin) |
map!(list_like, Tok::ListLike) |
- map_opt!(natural, |s| usize::from_str(s).ok().map(|n| Tok::Natural(n))) |
+ map!(natural, Tok::Natural) |
map!(integer, Tok::Integer) |
map!(identifier, |s: &str| Tok::Identifier(s.to_owned())) |
+ map!(string_lit, Tok::Text) |
value!(Tok::BraceL, tag!("{")) |
value!(Tok::BraceR, tag!("}")) |
@@ -305,4 +380,10 @@ fn test_lex() {
let lexer = Lexer::new(s);
let tokens = lexer.map(|r| r.unwrap().1).collect::<Vec<_>>();
assert_eq!(&tokens, &expected);
+
+ assert_eq!(string_lit(r#""a\&b""#).to_result(), Ok("ab".to_owned()));
+ assert_eq!(string_lit(r#""a\ \b""#).to_result(), Ok("ab".to_owned()));
+ assert!(string_lit(r#""a\ b""#).is_err());
+ assert_eq!(string_lit(r#""a\nb""#).to_result(), Ok("a\nb".to_owned()));
+ assert_eq!(string_lit(r#""\o141\x62\99""#).to_result(), Ok("abc".to_owned()));
}