aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/lux/lexer.clj14
-rw-r--r--src/lux/reader.clj21
2 files changed, 10 insertions, 25 deletions
diff --git a/src/lux/lexer.clj b/src/lux/lexer.clj
index 772ce2353..cbb351703 100644
--- a/src/lux/lexer.clj
+++ b/src/lux/lexer.clj
@@ -52,8 +52,13 @@
:else
(assert false (str "[Lexer Error] Unknown escape character: " escaped))))
+(defn ^:private escape-unicode [unicode]
+ (str (char (Integer/valueOf (.substring unicode 2) 16))))
+
(defn ^:private clean-line [raw-line]
- (string/replace raw-line #"\\." escape-char*))
+ (-> raw-line
+ (string/replace #"\\u[0-9a-fA-F]{4}" escape-unicode)
+ (string/replace #"\\." escape-char*)))
(defn ^:private lex-text-body [offset]
(|do [[_ eol? ^String pre-quotes] (&reader/read-regex #"^([^\"]*)")
@@ -85,9 +90,8 @@
_ (&reader/read-text "\"")]
(return (&/T [meta ($Text token)]))))
-(def ^:private +ident-re+ #"^([a-zA-Z\-\+\_\=!@$%^&*<>\.,/\\\|'`:\~\?][0-9a-zA-Z\-\+\_\=!@$%^&*<>\.,/\\\|'`:\~\?]*)"
- ;; #"^([^0-9\[\]\(\)\{\};#\s\"][^\[\]\(\)\{\};#\s\"]*)"
- )
+(def ^:private +ident-re+
+ #"^([^0-9\[\]\{\}\(\)\s\"#;][^\[\]\{\}\(\)\s\"#;]*)")
;; [Lexers]
(def ^:private lex-white-space
@@ -128,6 +132,8 @@
(|do [[meta _ _] (&reader/read-text "#\"")
token (&/try-all% (&/|list (|do [[_ _ escaped] (&reader/read-regex #"^(\\.)")]
(escape-char escaped))
+ (|do [[_ _ ^String unicode] (&reader/read-regex #"^(\\u[0-9a-fA-F]{4})")]
+ (return (str (char (Integer/valueOf (.substring unicode 2) 16)))))
(|do [[_ _ char] (&reader/read-regex #"^(.)")]
(return char))))
_ (&reader/read-text "\"")]
diff --git a/src/lux/reader.clj b/src/lux/reader.clj
index 72d0ee11b..f0509ec19 100644
--- a/src/lux/reader.clj
+++ b/src/lux/reader.clj
@@ -55,15 +55,6 @@
(when (.find matcher)
(.group matcher 0))))
-(defn ^:private re-find3! [^java.util.regex.Pattern regex column ^String line]
- (let [matcher (doto (.matcher regex line)
- (.region column (.length line))
- (.useAnchoringBounds true))]
- (when (.find matcher)
- (list (.group matcher 0)
- (.group matcher 1)
- (.group matcher 2)))))
-
;; [Exports]
(defn read-regex [regex]
(with-line
@@ -77,18 +68,6 @@
(&/T [(&/T [file-name line-num column-num*]) line]))))
($No (str "[Reader Error] Pattern failed: " regex))))))
-(defn read-regex2 [regex]
- (with-line
- (fn [file-name line-num column-num ^String line]
- (if-let [[^String match tok1 tok2] (re-find3! regex column-num line)]
- (let [match-length (.length match)
- column-num* (+ column-num match-length)]
- (if (= column-num* (.length line))
- ($Done (&/T [(&/T [file-name line-num column-num]) true (&/T [tok1 tok2])]))
- ($Yes (&/T [(&/T [file-name line-num column-num]) false (&/T [tok1 tok2])])
- (&/T [(&/T [file-name line-num column-num*]) line]))))
- ($No (str "[Reader Error] Pattern failed: " regex))))))
-
(defn read-regex+ [regex]
(with-lines
(fn [reader]