feat!: add AsyncPipe abstraction, significantly improve throughput of Transform streams

test/Test/Main.purs

@@ -3,18 +3,19 @@ module Test.Main where
 import Prelude
 
 import Data.Maybe (Maybe(..))
+import Data.Time.Duration (Milliseconds(..))
 import Effect (Effect)
 import Effect.Aff (launchAff_)
-import Test.Pipes.Node.Stream as Test.Pipes.Node.Stream
-import Test.Pipes.Node.Buffer as Test.Pipes.Node.Buffer
-import Test.Pipes.Node.FS as Test.Pipes.Node.FS
 import Test.Pipes.Collect as Test.Pipes.Collect
 import Test.Pipes.Construct as Test.Pipes.Construct
+import Test.Pipes.Node.Buffer as Test.Pipes.Node.Buffer
+import Test.Pipes.Node.FS as Test.Pipes.Node.FS
+import Test.Pipes.Node.Stream as Test.Pipes.Node.Stream
 import Test.Spec.Reporter (specReporter)
 import Test.Spec.Runner (defaultConfig, runSpec')
 
 main :: Effect Unit
-main = launchAff_ $ runSpec' (defaultConfig { exit = false, timeout = Nothing }) [ specReporter ] do
+main = launchAff_ $ runSpec' (defaultConfig { slow = Milliseconds 0.0, failFast = true, exit = false, timeout = Nothing }) [ specReporter ] do
   Test.Pipes.Node.Stream.spec
   Test.Pipes.Node.Buffer.spec
   Test.Pipes.Node.FS.spec

test/Test/Pipes.Node.Stream.js

@@ -1,4 +1,16 @@
 import Stream from 'stream'
+import * as CBOR from "cbor-x";
+import * as CSVDecode from "csv-parse";
+import * as CSVEncode from "csv-stringify";
+
+export const cborDecode = () => new CBOR.DecoderStream({useRecords: false, allowHalfOpen: true});
+export const cborEncode = () => new CBOR.EncoderStream({useRecords: false, allowHalfOpen: true});
+
+export const cborDecodeSync = a => () => CBOR.decodeMultiple(a);
+export const cborEncodeSync = a => () => CBOR.encode(a, {useRecords: false});
+
+export const csvDecode = () => CSVDecode.parse({columns: true, allowHalfOpen: true})
+export const csvEncode = () => CSVEncode.stringify({header: true, allowHalfOpen: true})
 
 export const discardTransform = () => new Stream.Transform({
   transform: function(_ck, _enc, cb) {
@@ -7,6 +19,16 @@ export const discardTransform = () => new Stream.Transform({
   objectMode: true
 })
 
+export const slowTransform = () => {
+  return new Stream.Transform({
+  transform: function(ck, _enc, cb) {
+    this.push(ck)
+    setTimeout(() => cb(), 4)
+  },
+  objectMode: true
+})
+}
+
 export const charsTransform = () => new Stream.Transform({
   transform: function(ck, _enc, cb) {
     ck.split('').filter(s => !!s).forEach(s => this.push(s))

test/Test/Pipes.Node.Stream.purs

@@ -2,49 +2,72 @@ module Test.Pipes.Node.Stream where
 
 import Prelude
 
-import Control.Monad.Trans.Class (lift)
+import Control.Monad.Cont (lift)
+import Control.Monad.Error.Class (liftEither)
+import Control.Monad.Except (runExcept)
 import Data.Array as Array
-import Data.Foldable (fold)
+import Data.Bifunctor (lmap)
+import Data.Either (Either(..))
+import Data.Foldable (fold, intercalate)
+import Data.FoldableWithIndex (forWithIndex_)
+import Data.FunctorWithIndex (mapWithIndex)
+import Data.Int as Int
 import Data.List ((:))
 import Data.List as List
 import Data.Maybe (Maybe)
 import Data.Newtype (wrap)
+import Data.Profunctor.Strong (first)
+import Data.String as String
 import Data.String.Gen (genAlphaString)
+import Data.Traversable (for_, traverse)
 import Data.Tuple.Nested (type (/\), (/\))
 import Effect (Effect)
 import Effect.Aff (Aff, delay)
 import Effect.Class (class MonadEffect, liftEffect)
+import Effect.Exception (error)
+import Effect.Unsafe (unsafePerformEffect)
+import Foreign (Foreign)
 import Node.Buffer (Buffer)
 import Node.Buffer as Buffer
 import Node.Encoding (Encoding(..))
 import Node.FS.Stream as FS.Stream
 import Node.FS.Sync as FS
 import Node.Stream.Object as O
-import Node.Zlib as Zlib
-import Pipes (each) as Pipes
+import Pipes (each) as Pipe
 import Pipes (yield, (>->))
+import Pipes.Async (sync, (>-/->))
+import Pipes.Collect as Pipe.Collect
 import Pipes.Core (Consumer, Producer, runEffect)
-import Pipes.Node.Buffer as Pipes.Buffer
-import Pipes.Node.Stream as S
-import Pipes.Prelude (mapFoldable, toListM) as Pipes
-import Simple.JSON (writeJSON)
-import Test.Common (jsonParse, jsonStringify, tmpFile, tmpFiles)
+import Pipes.Node.Buffer as Pipe.Buffer
+import Pipes.Node.FS as Pipe.FS
+import Pipes.Node.Stream as Pipe.Node
+import Pipes.Node.Zlib as Pipe.Zlib
+import Pipes.Prelude (toListM) as Pipe
+import Simple.JSON (readImpl, readJSON, writeJSON)
+import Test.Common (jsonStringify, tmpFile, tmpFiles)
 import Test.QuickCheck.Arbitrary (arbitrary)
 import Test.QuickCheck.Gen (randomSample')
 import Test.Spec (Spec, around, describe, it)
-import Test.Spec.Assertions (shouldEqual)
+import Test.Spec.Assertions (fail, shouldEqual)
 
 foreign import readableFromArray :: forall @a. Array a -> O.Readable a
 foreign import discardTransform :: forall a b. Effect (O.Transform a b)
+foreign import slowTransform :: forall a b. Effect (O.Transform a b)
 foreign import charsTransform :: Effect (O.Transform String String)
+foreign import cborEncodeSync :: forall a. a -> Effect Buffer
+foreign import cborDecodeSync :: forall a. Buffer -> Effect a
+foreign import cborEncode :: forall a. Effect (O.Transform a Buffer)
+foreign import cborDecode :: forall a. Effect (O.Transform Buffer a)
+foreign import csvEncode :: forall a. Effect (O.Transform a String)
+foreign import csvDecode :: forall a. Effect (O.Transform String a)
 
 writer :: forall m. MonadEffect m => String -> m (O.Writable Buffer /\ Consumer (Maybe Buffer) Aff Unit)
 writer a = do
   stream <- liftEffect $ O.unsafeCoerceWritable <$> FS.Stream.createWriteStream a
-  pure $ stream /\ S.fromWritable stream
+  pure $ stream /\ Pipe.Node.fromWritable stream
 
 reader :: forall m. MonadEffect m => String -> m (Producer (Maybe Buffer) Aff Unit)
-reader a = liftEffect $ S.fromReadable <$> O.unsafeCoerceReadable <$> FS.Stream.createReadStream a
+reader a = liftEffect $ Pipe.Node.fromReadable <$> O.unsafeCoerceReadable <$> FS.Stream.createReadStream a
 
 spec :: Spec Unit
 spec =
@@ -52,30 +75,30 @@ spec =
     describe "Readable" do
       describe "Readable.from(<Iterable>)" do
         it "empty" do
-          vals <- Pipes.toListM $ (S.fromReadable $ readableFromArray @{ foo :: String } []) >-> S.unEOS
+          vals <- Pipe.toListM $ (Pipe.Node.fromReadable $ readableFromArray @{ foo :: String } []) >-> Pipe.Node.unEOS
           vals `shouldEqual` List.Nil
         it "singleton" do
-          vals <- Pipes.toListM $ (S.fromReadable $ readableFromArray @{ foo :: String } [ { foo: "1" } ]) >-> S.unEOS
+          vals <- Pipe.toListM $ (Pipe.Node.fromReadable $ readableFromArray @{ foo :: String } [ { foo: "1" } ]) >-> Pipe.Node.unEOS
           vals `shouldEqual` ({ foo: "1" } : List.Nil)
         it "many elements" do
           let exp = (\n -> { foo: show n }) <$> Array.range 0 100
-          vals <- Pipes.toListM $ (S.fromReadable $ readableFromArray exp) >-> S.unEOS
+          vals <- Pipe.toListM $ (Pipe.Node.fromReadable $ readableFromArray exp) >-> Pipe.Node.unEOS
           vals `shouldEqual` (List.fromFoldable exp)
     describe "Writable" $ around tmpFile do
       describe "fs.WriteStream" do
         it "pipe to file" \p -> do
           stream <- O.unsafeCoerceWritable <$> liftEffect (FS.Stream.createWriteStream p)
           let
-            w = S.fromWritable stream
+            w = Pipe.Node.fromWritable stream
             source = do
               buf <- liftEffect $ Buffer.fromString "hello" UTF8
               yield buf
-          runEffect $ S.withEOS source >-> w
+          runEffect $ Pipe.Node.withEOS source >-> w
           contents <- liftEffect $ FS.readTextFile UTF8 p
           contents `shouldEqual` "hello"
           shouldEqual true =<< liftEffect (O.isWritableEnded stream)
         it "async pipe to file" \p -> do
-          w <- S.fromWritable <$> O.unsafeCoerceWritable <$> liftEffect (FS.Stream.createWriteStream p)
+          w <- Pipe.Node.fromWritable <$> O.unsafeCoerceWritable <$> liftEffect (FS.Stream.createWriteStream p)
           let
             source = do
               yield "hello, "
@@ -87,7 +110,7 @@ spec =
               yield "this is a "
               lift $ delay $ wrap 5.0
               yield "test."
-          runEffect $ S.withEOS (source >-> Pipes.Buffer.fromString UTF8) >-> w
+          runEffect $ Pipe.Node.withEOS (source >-> Pipe.Buffer.fromString UTF8) >-> w
           contents <- liftEffect $ FS.readTextFile UTF8 p
           contents `shouldEqual` "hello, world! this is a test."
         it "chained pipes" \p -> do
@@ -101,40 +124,96 @@ spec =
           let
             exp = fold (writeJSON <$> objs)
           stream /\ w <- liftEffect $ writer p
-          runEffect $ S.withEOS (Pipes.each objs >-> jsonStringify >-> Pipes.Buffer.fromString UTF8) >-> w
+          runEffect $ Pipe.Node.withEOS (Pipe.each objs >-> jsonStringify >-> Pipe.Buffer.fromString UTF8) >-> w
           contents <- liftEffect $ FS.readTextFile UTF8 p
           contents `shouldEqual` exp
           shouldEqual true =<< liftEffect (O.isWritableEnded stream)
     describe "Transform" do
-      it "gzip" do
-        let
-          json = yield $ writeJSON { foo: "bar" }
-          exp = "1f8b0800000000000003ab564acbcf57b2524a4a2c52aa0500eff52bfe0d000000"
-        gzip <- S.fromTransform <$> O.unsafeCoerceTransform <$> liftEffect (Zlib.toDuplex <$> Zlib.createGzip)
-        outs :: List.List String <- Pipes.toListM (S.withEOS (json >-> Pipes.Buffer.fromString UTF8) >-> gzip >-> S.unEOS >-> Pipes.Buffer.toString Hex)
-        fold outs `shouldEqual` exp
-      around tmpFiles
-        $ it "file >-> gzip >-> file >-> gunzip" \(a /\ b) -> do
-            liftEffect $ FS.writeTextFile UTF8 a $ writeJSON [ 1, 2, 3, 4 ]
-            areader <- liftEffect $ reader a
-            bwritestream /\ bwriter <- liftEffect $ writer b
-            gzip <- S.fromTransform <$> O.unsafeCoerceTransform <$> liftEffect (Zlib.toDuplex <$> Zlib.createGzip)
-            runEffect $ areader >-> gzip >-> bwriter
-            shouldEqual true =<< liftEffect (O.isWritableEnded bwritestream)
+      let
+        bignums = Array.range 1 1000
+        firstNames = String.split (wrap "\n") $ unsafePerformEffect (FS.readTextFile UTF8 "./test/Test/first_names.txt")
+        lastNames = String.split (wrap "\n") $ unsafePerformEffect (FS.readTextFile UTF8 "./test/Test/last_names.txt")
+        names n = do
+          first <- firstNames
+          last <- Array.take (Int.round $ Int.toNumber n / Int.toNumber (Array.length firstNames)) lastNames
+          pure $ first <> " " <> last
+        people n = mapWithIndex (\ix name -> {id: show $ ix + 1, name}) (names n)
+        peopleCSV n = "id,name\n" <> intercalate "\n" ((\{id, name} -> id <> "," <> name) <$> people n)
 
-            gunzip <- S.fromTransform <$> O.unsafeCoerceTransform <$> liftEffect (Zlib.toDuplex <$> Zlib.createGunzip)
-            breader <- liftEffect $ reader b
-            nums <- Pipes.toListM (breader >-> gunzip >-> S.unEOS >-> Pipes.Buffer.toString UTF8 >-> jsonParse @(Array Int) >-> Pipes.mapFoldable identity)
-            Array.fromFoldable nums `shouldEqual` [ 1, 2, 3, 4 ]
+      for_ [4000, 8000, 32000, 64000, 200000] \n -> do
+        let
+          csv = peopleCSV n
+          people' = people n
+        around tmpFiles
+          $ it (show n <> " row csv >-/-> csv-parse >-/-> cborEncode") \(a /\ _) -> do
+              liftEffect $ FS.writeTextFile UTF8 a csv
+              cbor :: Buffer <- Pipe.Collect.toBuffer
+                $ Pipe.FS.read a
+                  >-> Pipe.Node.inEOS (Pipe.Buffer.toString UTF8)
+                  >-/-> Pipe.Node.fromTransform csvDecode
+                  >-/-> Pipe.Node.fromTransform cborEncode
+                  >-> Pipe.Node.unEOS
+              f :: Array Foreign <- liftEffect $ cborDecodeSync cbor
+              ppl <- traverse (liftEither <<< lmap (error <<< show) <<< runExcept <<< readImpl) f
+              ppl `shouldEqual` people'
+
+        around tmpFiles
+          $ it (show n <> " row csv >-> sync csv-parse >-> sync cborEncode") \(a /\ _) -> do
+              liftEffect $ FS.writeTextFile UTF8 a csv
+              cbor :: Buffer <- Pipe.Collect.toBuffer
+                $ Pipe.FS.read a
+                  >-> Pipe.Node.inEOS (Pipe.Buffer.toString UTF8)
+                  >-> sync (Pipe.Node.fromTransform csvDecode)
+                  >-> sync (Pipe.Node.fromTransform cborEncode)
+                  >-> Pipe.Node.unEOS
+              f :: Array Foreign <- liftEffect $ cborDecodeSync cbor
+              ppl <- traverse (liftEither <<< lmap (error <<< show) <<< runExcept <<< readImpl) f
+              ppl `shouldEqual` people'
+
+      around tmpFiles
+        $ it "file >-> sync gzip >-> sync gunzip" \(a /\ _) -> do
+            liftEffect $ FS.writeTextFile UTF8 a $ writeJSON bignums
+            json <- Pipe.Collect.toMonoid
+              $ Pipe.FS.read a
+                >-> sync Pipe.Zlib.gzip
+                >-> sync Pipe.Zlib.gunzip
+                >-> Pipe.Node.unEOS
+                >-> Pipe.Buffer.toString UTF8
+            readJSON json `shouldEqual` (Right bignums)
+
+      around tmpFiles
+        $ it "file >-/-> gzip >-/-> slow >-/-> gunzip" \(a /\ _) -> do
+            liftEffect $ FS.writeTextFile UTF8 a $ writeJSON bignums
+            json <-
+              Pipe.Collect.toMonoid
+              $ Pipe.FS.read a
+                >-/-> Pipe.Zlib.gzip
+                >-/-> Pipe.Node.fromTransform slowTransform
+                >-/-> Pipe.Zlib.gunzip
+                >-> Pipe.Node.unEOS
+                >-> Pipe.Buffer.toString UTF8
+
+            readJSON json `shouldEqual` (Right bignums)
+      around tmpFiles
+        $ it "file >-> sync gzip >-> sync slow >-> sync gunzip" \(a /\ _) -> do
+            liftEffect $ FS.writeTextFile UTF8 a $ writeJSON bignums
+            json <-
+              Pipe.Collect.toMonoid
+              $ Pipe.FS.read a
+                >-> sync Pipe.Zlib.gzip
+                >-> sync (Pipe.Node.fromTransform slowTransform)
+                >-> sync Pipe.Zlib.gunzip
+                >-> Pipe.Node.unEOS
+                >-> Pipe.Buffer.toString UTF8
+
+            readJSON json `shouldEqual` (Right bignums)
       around tmpFile $ it "file >-> discardTransform" \(p :: String) -> do
         liftEffect $ FS.writeTextFile UTF8 p "foo"
         r <- reader p
-        discard' <- liftEffect discardTransform
-        out :: List.List Int <- Pipes.toListM $ r >-> S.fromTransform discard' >-> S.unEOS
+        out :: List.List Int <- Pipe.toListM $ r >-/-> Pipe.Node.fromTransform discardTransform >-> Pipe.Node.unEOS
         out `shouldEqual` List.Nil
       around tmpFile $ it "file >-> charsTransform" \(p :: String) -> do
         liftEffect $ FS.writeTextFile UTF8 p "foo bar"
         r <- reader p
-        chars' <- liftEffect charsTransform
-        out :: List.List String <- Pipes.toListM $ r >-> S.inEOS (Pipes.Buffer.toString UTF8) >-> S.fromTransform chars' >-> S.unEOS
+        out :: List.List String <- Pipe.toListM $ r >-> Pipe.Node.inEOS (Pipe.Buffer.toString UTF8) >-/-> Pipe.Node.fromTransform charsTransform >-> Pipe.Node.unEOS
         out `shouldEqual` List.fromFoldable [ "f", "o", "o", " ", "b", "a", "r" ]