support backlash commentsw

diff --git a/hylo-lang/hyloc/src/tokenizer.rs b/hylo-lang/hyloc/src/tokenizer.rs
index 0240aa2eb58a68963c9053534ba5f233979cf5ad..9ac92a53a5a6609ea6cfc061cc8a4ecf3c34a9f9 100644 (file)
--- a/hylo-lang/hyloc/src/tokenizer.rs
+++ b/hylo-lang/hyloc/src/tokenizer.rs
@@ -62,9 +62,10 @@ pub fn tokenize<'a>(input: &'a str) -> Result<Vec<Token<'a>>> {
     let mut result = vec![];
     let mut string_start: Option<usize> = None;
     let mut word_or_num_start: Option<usize> = None;
-    let mut last_is_escape = false;
+    let mut last_is_backslash = false;
     let mut last_is_whitespace = true;
-    let mut in_comment = false;
+    let mut in_doc_comment = false;
+    let mut in_line_comment = false;
     let mut index = 0;
     let mut first_char = true;
 
@@ -76,48 +77,60 @@ pub fn tokenize<'a>(input: &'a str) -> Result<Vec<Token<'a>>> {
             index += 1;
         }
         
-        if in_comment {
+        if in_doc_comment {
             if char == ')' {
-                in_comment = false;
+                in_doc_comment = false;
                 last_is_whitespace = true; // not really true, but means don't need space after
             }
             continue;
         }
 
+        if in_line_comment {
+            if char == '\n' {
+                in_line_comment = false;
+                last_is_whitespace = true; // not really true, but means don't need space after
+            }
+        }
+
         if char == '"' {
             if let Some(start) = string_start {
-                if !last_is_escape {
+                if !last_is_backslash {
                     result.push(Token::String(&input[start..index]));
                     string_start = None;
                 }
             } else {
                 string_start = Some(index + 1)                
             }
-            last_is_escape = false;
+            last_is_backslash = false;
             last_is_whitespace = false;
             continue;
         }
 
-        last_is_escape = char == '\\';
 
         if string_start.is_some() {
+            last_is_backslash = char == '\\';
             continue;
         }
 
         if char.is_whitespace() {
-            if !last_is_whitespace && let Some(start) = word_or_num_start {
+            if last_is_backslash {
+                in_line_comment = true;                
+            } else if !last_is_whitespace && let Some(start) = word_or_num_start {
                 let token = &input[start..index];
                 if token == "(" {
-                    in_comment = true;
+                    in_doc_comment = true;
                 } else {
                     result.push(Token::parse_word_or_num(&input[start..index])?);
                 }
                 word_or_num_start = None;
             }
             last_is_whitespace = true;
+            last_is_backslash = false;
             continue;
         }
 
+        last_is_backslash = char == '\\';
+
         if index == input.len() - 1 {
             if !last_is_whitespace && let Some(start) = word_or_num_start {
                 result.push(Token::parse_word_or_num(&input[start..])?);
@@ -140,6 +153,8 @@ mod tests {
     #[test]
     fn try_some_tokenizing() {
         let result = tokenize("
+
+        \\ soup
             2 3.4 - -88 bacon \"hello\" 43:f32 2345:u32 -57:i8 soup
 ");
         println!("result: {:?}", result);