More efficient binary parsing.

5 files changed, 80 insertions, 45 deletions

diff --git a/stdlib/source/library/lux/control/concurrency/thread.lux b/stdlib/source/library/lux/control/concurrency/thread.lux
index f8d92bc77..619b9733d 100644
--- a/stdlib/source/library/lux/control/concurrency/thread.lux
+++ b/stdlib/source/library/lux/control/concurrency/thread.lux
@@ -170,7 +170,7 @@
                 (if started?
                   (in [])
                   (do !
-                    [_ (atom.has! true ..started?)]
+                    [_ (atom.write! true ..started?)]
                     (loop (again [_ []])
                       (do !
                         [threads (atom.read! ..runner)]
diff --git a/stdlib/source/library/lux/control/parser/binary.lux b/stdlib/source/library/lux/control/parser/binary.lux
index a1c565360..1e360a194 100644
--- a/stdlib/source/library/lux/control/parser/binary.lux
+++ b/stdlib/source/library/lux/control/parser/binary.lux
@@ -1,6 +1,7 @@
 (.using
  [library
   [lux {"-" and or nat int rev list type symbol}
+   [ffi {"+"}]
    [type {"+" sharing}]
    [abstract
     [hash {"+" Hash}]
@@ -9,7 +10,8 @@
     ["[0]" try {"+" Try}]
     ["[0]" exception {"+" exception:}]]
    [data
-    ["/" binary {"+" Binary}]
+    ["/" binary "_"
+     ["[1]" \\unsafe {"+" Binary}]]
     [text
      ["%" format {"+" format}]
      [encoding
@@ -17,7 +19,9 @@
     [collection
      ["[0]" list]
      ["[0]" sequence {"+" Sequence}]
-     ["[0]" set {"+" Set}]]]
+     ["[0]" set {"+" Set}]
+     [array
+      [\\unsafe {"+"}]]]]
    [macro
     ["^" pattern]
     ["[0]" template]]
@@ -39,22 +43,31 @@
    "Binary length" (%.nat binary_length)
    "Bytes read" (%.nat bytes_read)))
 
+(template [<name> <extension>]
+  [(template: (<name> <parameter> <subject>)
+     [(<extension> <parameter> <subject>)])]
+
+  [n#= "lux i64 ="]
+  [n#+ "lux i64 +"]
+  [n#- "lux i64 -"]
+  )
+
 (def: .public (result parser input)
   (All (_ a) (-> (Parser a) Binary (Try a)))
   (case (parser [0 input])
-    {try.#Failure msg}
-    {try.#Failure msg}
-    
     {try.#Success [[end _] output]}
     (let [length (/.size input)]
-      (if (n.= end length)
+      (if (n#= end length)
         {try.#Success output}
-        (exception.except ..binary_was_not_fully_read [length end])))))
+        (exception.except ..binary_was_not_fully_read [length end])))
+
+    failure
+    (as_expected failure)))
 
 (def: .public end?
   (Parser Bit)
   (function (_ (^.let input [offset data]))
-    {try.#Success [input (n.= offset (/.size data))]}))
+    {try.#Success [input (n#= offset (/.size data))]}))
 
 (def: .public offset
   (Parser Offset)
@@ -64,7 +77,7 @@
 (def: .public remaining
   (Parser Nat)
   (function (_ (^.let input [offset data]))
-    {try.#Success [input (n.- offset (/.size data))]}))
+    {try.#Success [input (n#- offset (/.size data))]}))
 
 (type: .public Size
   Nat)
@@ -74,16 +87,24 @@
 (def: .public size_32 Size (n.* 2 size_16))
 (def: .public size_64 Size (n.* 2 size_32))
 
+(exception: .public (range_out_of_bounds [length Nat
+                                          start Nat
+                                          end Nat])
+  (exception.report
+   "Length" (%.nat length)
+   "Range start" (%.nat start)
+   "Range end" (%.nat end)))
+
 (template [<name> <size> <read>]
   [(def: .public <name>
      (Parser I64)
-     (function (_ [offset binary])
-       (case (<read> offset binary)
-         {try.#Success data}
-         {try.#Success [(n.+ <size> offset) binary] data}
-         
-         {try.#Failure error}
-         {try.#Failure error})))]
+     (function (_ [start binary])
+       (let [end (n#+ <size> start)]
+         (if (n.< end (/.size binary))
+           (exception.except ..range_out_of_bounds [(/.size binary) start end])
+           (|> (<read> start binary)
+               [[end binary]]
+               {try.#Success})))))]
 
   [bits_8  ..size_8  /.bits_8]
   [bits_16 ..size_16 /.bits_16]
@@ -153,12 +174,16 @@
 
 (def: .public (segment size)
   (-> Nat (Parser Binary))
-  (function (_ [offset binary])
-    (case size
-      0 {try.#Success [[offset binary] (/.empty 0)]}
-      _ (|> binary
-            (/.slice offset size)
-            (# try.monad each (|>> [[(n.+ size offset) binary]]))))))
+  (case size
+    0 (//#in (/.empty 0))
+    _ (function (_ [start binary])
+        (let [end (n#+ size start)]
+          (if (n.< end (/.size binary))
+            (exception.except ..range_out_of_bounds [(/.size binary) start end])
+            (|> binary
+                (/.slice start size)
+                [[end binary]]
+                {try.#Success}))))))
 
 (template [<size> <name> <bits>]
   [(`` (def: .public <name>
@@ -232,7 +257,7 @@
     [raw (..list value)
      .let [output (set.of_list hash raw)]
      _ (//.assertion (exception.error ..set_elements_are_not_unique [])
-                     (n.= (list.size raw)
+                     (n#= (list.size raw)
                           (set.size output)))]
     (in output)))
 
diff --git a/stdlib/source/library/lux/data/binary.lux b/stdlib/source/library/lux/data/binary.lux
index 6615131c5..6ccbb0442 100644
--- a/stdlib/source/library/lux/data/binary.lux
+++ b/stdlib/source/library/lux/data/binary.lux
@@ -31,7 +31,7 @@
   (-> Nat Binary)
   (|>> /.empty))
 
-(def: .public (aggregate $ init it)
+(def: .public (mix $ init it)
   (All (_ a) (-> (-> I64 a a) a Binary a))
   (let [size (/.size it)]
     (loop (again [index 0
diff --git a/stdlib/source/library/lux/data/format/tar.lux b/stdlib/source/library/lux/data/format/tar.lux
index afea5a3dd..4fb3522a6 100644
--- a/stdlib/source/library/lux/data/format/tar.lux
+++ b/stdlib/source/library/lux/data/format/tar.lux
@@ -172,7 +172,7 @@
 
   (def: checksum
     (-> Binary Nat)
-    (binary.aggregate n.+ 0))
+    (binary.mix n.+ 0))
 
   (def: checksum_checksum
     (|> ..dummy_checksum
@@ -217,10 +217,10 @@
 (def: ascii?
   (-> Text Bit)
   (|>> (# utf8.codec encoded)
-       (binary.aggregate (function (_ char verdict)
-                           (.and verdict
-                                 (n.<= ..last_ascii char)))
-                         true)))
+       (binary.mix (function (_ char verdict)
+                     (.and verdict
+                           (n.<= ..last_ascii char)))
+                   true)))
 
 (exception: .public (not_ascii [text Text])
   (exception.report
diff --git a/stdlib/source/library/lux/tool/compiler/language/lux/phase/extension/analysis/ruby.lux b/stdlib/source/library/lux/tool/compiler/language/lux/phase/extension/analysis/ruby.lux
index 32c57de4e..ab9ed4837 100644
--- a/stdlib/source/library/lux/tool/compiler/language/lux/phase/extension/analysis/ruby.lux
+++ b/stdlib/source/library/lux/tool/compiler/language/lux/phase/extension/analysis/ruby.lux
@@ -9,7 +9,7 @@
      ["<[0]>" code {"+" Parser}]]]
    [data
     [collection
-     ["[0]" array {"+" Array}]
+     ["[0]" array]
      ["[0]" dictionary]
      ["[0]" list]]]
    ["[0]" type
@@ -32,11 +32,13 @@
    [<code>.any
     (function (_ extension phase archive lengthC)
       (<| analysis/type.with_var
-          (function (_ [@var :var:]))
+          (function (_ [@read :read:]))
+          analysis/type.with_var
+          (function (_ [@write :write:]))
           (do phase.monad
             [lengthA (<| (analysis/type.expecting Nat)
                          (phase archive lengthC))
-             _ (analysis/type.inference (type (Array :var:)))]
+             _ (analysis/type.inference (type (array.Array' :read: :write:)))]
             (in {analysis.#Extension extension (list lengthA)}))))]))
 
 (def: array::length
@@ -45,9 +47,11 @@
    [<code>.any
     (function (_ extension phase archive arrayC)
       (<| analysis/type.with_var
-          (function (_ [@var :var:]))
+          (function (_ [@read :read:]))
+          analysis/type.with_var
+          (function (_ [@write :write:]))
           (do phase.monad
-            [arrayA (<| (analysis/type.expecting (type (Array :var:)))
+            [arrayA (<| (analysis/type.expecting (type (array.Array' :read: :write:)))
                         (phase archive arrayC))
              _ (analysis/type.inference Nat)]
             (in {analysis.#Extension extension (list arrayA)}))))]))
@@ -58,13 +62,15 @@
    [(<>.and <code>.any <code>.any)
     (function (_ extension phase archive [indexC arrayC])
       (<| analysis/type.with_var
-          (function (_ [@var :var:]))
+          (function (_ [@read :read:]))
+          analysis/type.with_var
+          (function (_ [@write :write:]))
           (do phase.monad
             [indexA (<| (analysis/type.expecting Nat)
                         (phase archive indexC))
-             arrayA (<| (analysis/type.expecting (type (Array :var:)))
+             arrayA (<| (analysis/type.expecting (type (array.Array' :read: :write:)))
                         (phase archive arrayC))
-             _ (analysis/type.inference :var:)]
+             _ (analysis/type.inference :read:)]
             (in {analysis.#Extension extension (list indexA arrayA)}))))]))
 
 (def: array::write
@@ -73,15 +79,17 @@
    [($_ <>.and <code>.any <code>.any <code>.any)
     (function (_ extension phase archive [indexC valueC arrayC])
       (<| analysis/type.with_var
-          (function (_ [@var :var:]))
+          (function (_ [@read :read:]))
+          analysis/type.with_var
+          (function (_ [@write :write:]))
           (do phase.monad
             [indexA (<| (analysis/type.expecting Nat)
                         (phase archive indexC))
-             valueA (<| (analysis/type.expecting :var:)
+             valueA (<| (analysis/type.expecting :write:)
                         (phase archive valueC))
-             arrayA (<| (analysis/type.expecting (type (Array :var:)))
+             arrayA (<| (analysis/type.expecting (type (array.Array' :read: :write:)))
                         (phase archive arrayC))
-             _ (analysis/type.inference (type (Array :var:)))]
+             _ (analysis/type.inference (type (array.Array' :read: :write:)))]
             (in {analysis.#Extension extension (list indexA valueA arrayA)}))))]))
 
 (def: array::delete
@@ -90,13 +98,15 @@
    [($_ <>.and <code>.any <code>.any)
     (function (_ extension phase archive [indexC arrayC])
       (<| analysis/type.with_var
-          (function (_ [@var :var:]))
+          (function (_ [@read :read:]))
+          analysis/type.with_var
+          (function (_ [@write :write:]))
           (do phase.monad
             [indexA (<| (analysis/type.expecting Nat)
                         (phase archive indexC))
-             arrayA (<| (analysis/type.expecting (type (Array :var:)))
+             arrayA (<| (analysis/type.expecting (type (array.Array' :read: :write:)))
                         (phase archive arrayC))
-             _ (analysis/type.inference (type (Array :var:)))]
+             _ (analysis/type.inference (type (array.Array' :read: :write:)))]
             (in {analysis.#Extension extension (list indexA arrayA)}))))]))
 
 (def: bundle::array