chore: prepare v2.0.0

README.md

@@ -2,41 +2,114 @@
 
 Interact with node streams in object mode using [`Pipes`]!
 
-## Example
+## Install
-```purescript
-import Prelude
-
-import Effect.Aff (launchAff_)
-import Effect (Effect)
-import Effect.Class (liftEffect)
-import Effect.Console (log)
-import Pipes.Prelude ((>->))
-import Pipes.Prelude as Pipes
-import Pipes.Core as Pipes.Core
-import Pipes.Node.FS.Stream as FS
-import Pipes.Node.Zlib as Zlib
-import Pipes.CSV.Parse as CSV.Parse
-
--- == my-zipped-data.csv ==
--- id,foo,is_deleted
--- 1,hello,f
--- 2,goodbye,t
-
--- Logs:
--- {id: 1, foo: "hello", is_deleted: false}
--- {id: 2, foo: "goodbye", is_deleted: true}
-main :: Effect Unit
-main =
-  Pipes.Core.runEffect
-    $ FS.createReadStream "my-zipped-data.csv.gz"
-        >-> Zlib.gunzip
-        >-> CSV.Parse.parse @{id :: Int, foo :: String, is_deleted :: Boolean}
-        >-> Pipes.mapM (liftEffect <<< log)
-```
-
-## Installing
 ```bash
 spago install node-stream-pipes
 ```
 
+## Usage
+### Node Streams
+#### Raw Streams
+Raw `objectMode` Node streams are represented in `Node.Stream.Object`:
+ - `Writable a` accepts chunks of type `a`
+ - `Readable a` emits chunks of type `a`
+ - `Transform a b` transforms chunks from `a` to `b`
+
+Non-Object streams can also be represented with these types; for example an `fs.WriteStream`
+can be coerced to `Writable Buffer` without issue.
+
+Interop between these types and `Node.Stream` are provided in `Node.Stream.Object`:
+- `unsafeFrom{String,Buffer}{Writable,Readable,Transform}`
+- `unsafeCoerce{Writable,Readable,Transform}`
+
+#### Pipes
+Streams in `Node.Stream.Object` can be converted to `Producer`s, `Consumer`s and `Pipe`s with `Pipes.Node.Stream`:
+ - `fromReadable :: forall a. <Readable a> -> Producer (Maybe a) <Aff> Unit`
+ - `fromWritable :: forall a. <Writable a> -> Consumer (Maybe a) <Aff> Unit`
+ - `fromTransform :: forall a b. <Transform a b> -> Pipe (Maybe a) (Maybe b) <Aff> Unit`
+
+#### EOS Marker
+Normally, pipe computations will not be executed once any computation in a pipeline exits.
+
+To allow for resource cleanup and awareness that the stream is about to close,
+`Maybe a` is used occasionally in this package as an End-of-Stream marker:
+
+```purescript
+-- foo.txt is "hello, world!\n"
+chunks <- Pipes.Collect.toArray $ Pipes.FS.read "foo.txt" >-> Pipes.Node.Stream.inEOS (Pipes.Buffer.toString UTF8)
+chunks `shouldEqual` [Just "hello, world!\n", Nothing]
+```
+
+Pipes from `a -> b` unaware of EOS can be lifted to `Maybe a -> Maybe b` with `Pipes.Node.Stream.inEOS`.
+
+Producers of `Maybe a` can drop the EOS marker and emit `a` with `Pipes.Node.Stream.unEOS`.
+
+Producers of `a` can have an EOS marker added with `Pipes.Node.Stream.withEOS`.
+
+#### Example
+`Pipes.PassThrough.js`
+```javascript
+import {PassThrough} from 'stream'
+
+export const makePassThrough = () => new PassThrough()
+```
+
+`Pipes.PassThrough.purs`
+```purescript
+module Pipes.PassThrough where
+
+import Prelude
+
+import Effect (Effect)
+import Effect.Class (liftEffect)
+import Effect.Aff (Aff)
+import Pipes.Core (Pipe)
+import Node.Stream.Object as ObjectStream
+import Pipes.Node.Stream as Pipes.Node.Stream
+
+type PassThroughStream a = ObjectStream.Transform a a
+
+foreign import makeRaw :: Effect PassThroughStream
+
+passThrough :: forall a. Pipe a a Aff Unit
+passThrough = do
+  raw <- liftEffect $ makeRaw
+  Pipes.Node.Stream.fromTransform raw
+```
+
+### Utilities
+This package provides utilities that explicitly use `MonadRec` to ensure stack-safety
+when dealing with producers of large amounts of data.
+
+- `Pipes.Collect` provides stack-safe utilities for executing a pipeline and collecting results into a collection, `Buffer`, `Monoid` etc.
+- `Pipes.Construct` provides stack-safe utilities for creating producers from in-memory collections.
+- `Pipes.Util` provides some miscellaneous utilities missing from `pipes`.
+
+### Zlib
+Pipes for compression & decompression using `zlib` are provided in `Pipes.Node.Zlib`.
+
+### FS
+Read files with:
+- `Pipes.Node.FS.read <path>`
+- `Pipes.Node.FS.read' <WriteStreamOptions> <path>`
+
+```purescript
+Pipes.Collect.toStringWith UTF8 $ Pipes.Node.FS.read "foo.txt" >-> Pipes.Stream.unEOS
+```
+
+Write files with:
+ - `Pipes.Node.FS.write' <WriteStreamOptions> <path>`
+ - `Pipes.Node.FS.trunc <path>`
+ - `Pipes.Node.FS.create <path>`
+ - `Pipes.Node.FS.append <path>`
+
+```purescript
+  Pipes.Stream.withEOS (
+    Pipes.Construct.eachArray ["id,name", "1,henry", "2,suzie"]
+    >-> Pipes.Util.intersperse "\n"
+    >-> Pipes.Buffer.fromString UTF8
+  )
+  >-> Pipes.Node.FS.create "foo.csv"
+```
+
 [`Pipes`]: https://pursuit.purescript.org/packages/purescript-pipes/8.0.0

package.json

@@ -1,12 +1,13 @@
 {
   "name": "purescript-node-stream-pipes",
-  "version": "v1.4.1",
+  "version": "v2.0.0",
   "type": "module",
   "dependencies": {
-    "csv-parse": "^5.5.5",
+    "csv-parse": "^5.5.6",
-    "csv-stringify": "^6.4.6"
+    "csv-stringify": "^6.5.0"
   },
   "devDependencies": {
+    "cbor-x": "^1.5.9",
     "typescript": "^5.4.5"
   }
 }

spago.lock

@@ -5,11 +5,15 @@ workspace:
       dependencies:
         - aff: ">=7.1.0 <8.0.0"
         - arrays: ">=7.3.0 <8.0.0"
+        - console: ">=6.1.0 <7.0.0"
+        - control: ">=6.0.0 <7.0.0"
+        - datetime: ">=6.1.0 <7.0.0"
         - effect: ">=4.0.0 <5.0.0"
         - either: ">=6.1.0 <7.0.0"
         - exceptions: ">=6.0.0 <7.0.0"
         - foldable-traversable: ">=6.0.0 <7.0.0"
         - foreign-object: ">=4.1.0 <5.0.0"
+        - fork: ">=6.0.0 <7.0.0"
         - lists: ">=7.0.0 <8.0.0"
         - maybe: ">=6.0.0 <7.0.0"
         - mmorph: ">=7.0.0 <8.0.0"
@@ -20,6 +24,7 @@ workspace:
         - node-path: ">=5.0.0 <6.0.0"
         - node-streams: ">=9.0.0 <10.0.0"
         - node-zlib: ">=0.4.0 <0.5.0"
+        - now: ">=6.0.0 <7.0.0"
         - ordered-collections: ">=3.2.0 <4.0.0"
         - parallel: ">=6.0.0 <7.0.0"
         - pipes: ">=8.0.0 <9.0.0"

spago.yaml

@@ -1,7 +1,7 @@
 package:
   name: node-stream-pipes
   publish:
-    version: '1.4.1'
+    version: '2.0.0'
     license: 'GPL-3.0-or-later'
     location:
       githubOwner: 'cakekindel'
@@ -12,11 +12,15 @@ package:
   dependencies:
     - aff: ">=7.1.0 <8.0.0"
     - arrays: ">=7.3.0 <8.0.0"
+    - console: ">=6.1.0 <7.0.0"
+    - control: ">=6.0.0 <7.0.0"
+    - datetime: ">=6.1.0 <7.0.0"
     - effect: ">=4.0.0 <5.0.0"
     - either: ">=6.1.0 <7.0.0"
     - exceptions: ">=6.0.0 <7.0.0"
     - foldable-traversable: ">=6.0.0 <7.0.0"
     - foreign-object: ">=4.1.0 <5.0.0"
+    - fork: ">=6.0.0 <7.0.0"
     - lists: ">=7.0.0 <8.0.0"
     - maybe: ">=6.0.0 <7.0.0"
     - mmorph: ">=7.0.0 <8.0.0"
@@ -27,6 +31,7 @@ package:
     - node-path: ">=5.0.0 <6.0.0"
     - node-streams: ">=9.0.0 <10.0.0"
     - node-zlib: ">=0.4.0 <0.5.0"
+    - now: ">=6.0.0 <7.0.0"
     - ordered-collections: ">=3.2.0 <4.0.0"
     - parallel: ">=6.0.0 <7.0.0"
     - pipes: ">=8.0.0 <9.0.0"

src/Node.Stream.Object.js

@@ -21,6 +21,9 @@ export const isReadableEndedImpl = (s) => () => s.readableEnded;
 /** @type {(s: Stream.Writable | Stream.Transform) => () => boolean} */
 export const isWritableEndedImpl = (s) => () => s.writableEnded;
 
+/** @type {(s: Stream.Writable | Stream.Transform) => () => boolean} */
+export const isWritableFinishedImpl = (s) => () => s.writableFinished;
+
 /** @type {(s: Stream.Writable | Stream.Transform) => () => void} */
 export const endImpl = (s) => () => s.end();
 

src/Node.Stream.Object.purs

@@ -14,8 +14,9 @@ import Data.Maybe (Maybe(..))
 import Data.Show.Generic (genericShow)
 import Effect (Effect)
 import Effect.Aff (Aff, effectCanceler, makeAff)
+import Effect.Aff as Aff
 import Effect.Class (liftEffect)
-import Effect.Exception (Error)
+import Effect.Exception (Error, error)
 import Effect.Uncurried (mkEffectFn1)
 import Node.Buffer (Buffer)
 import Node.EventEmitter (EventHandle(..))
@@ -34,6 +35,10 @@ derive instance Eq a => Eq (ReadResult a)
 instance Show (ReadResult a) where
   show = genericShow <<< map (const "..")
 
+maybeReadResult :: forall a. ReadResult a -> Maybe a
+maybeReadResult (ReadWouldBlock) = Nothing
+maybeReadResult (ReadJust a) = Just a
+
 data WriteResult
   = WriteWouldBlock
   | WriteOk
@@ -57,6 +62,7 @@ foreign import isReadableImpl :: forall s. s -> Effect Boolean
 foreign import isWritableImpl :: forall s. s -> Effect Boolean
 foreign import isReadableEndedImpl :: forall s. s -> Effect Boolean
 foreign import isWritableEndedImpl :: forall s. s -> Effect Boolean
+foreign import isWritableFinishedImpl :: forall s. s -> Effect Boolean
 foreign import isClosedImpl :: forall s. s -> Effect Boolean
 foreign import needsDrainImpl :: forall s. s -> Effect Boolean
 foreign import readableLengthImpl :: forall s. s -> Effect Int
@@ -90,6 +96,7 @@ class Stream s <= Write s a | s -> a where
   isWritable :: s -> Effect Boolean
   needsDrain :: s -> Effect Boolean
   isWritableEnded :: s -> Effect Boolean
+  isWritableFinished :: s -> Effect Boolean
   write :: s -> a -> Effect WriteResult
   end :: s -> Effect Unit
 
@@ -112,18 +119,21 @@ else instance (Read s a) => Read s a where
 instance Write (Writable a) a where
   isWritable = isWritableImpl
   isWritableEnded = isWritableEndedImpl
+  isWritableFinished = isWritableFinishedImpl
   write s = writeImpl writeResultFFI s
   end = endImpl
   needsDrain = needsDrainImpl
 else instance Write (Transform a b) a where
   isWritable = isWritableImpl
   isWritableEnded = isWritableEndedImpl
+  isWritableFinished = isWritableFinishedImpl
   write s = writeImpl writeResultFFI s
   end = endImpl
   needsDrain = needsDrainImpl
 else instance (Write s a) => Write s a where
   isWritable = isWritableImpl
   isWritableEnded = isWritableEndedImpl
+  isWritableFinished = isWritableFinishedImpl
   write s a = write s a
   end s = end s
   needsDrain = needsDrainImpl
@@ -134,45 +144,74 @@ withErrorST s = do
   cancel <- flip (Event.once errorH) s \e -> void $ liftST $ STRef.write (Just e) error
   pure { error, cancel }
 
-fromBufferReadable :: forall r. Stream.Readable r -> Readable Buffer
+unsafeCoerceWritable :: forall r a. Stream.Writable r -> Writable a
-fromBufferReadable = unsafeCoerce
+unsafeCoerceWritable = unsafeCoerce
 
-fromBufferTransform :: Stream.Duplex -> Transform Buffer Buffer
+unsafeCoerceReadable :: forall r a. Stream.Readable r -> Readable a
-fromBufferTransform = unsafeCoerce
+unsafeCoerceReadable = unsafeCoerce
 
-fromBufferWritable :: forall r. Stream.Writable r -> Writable Buffer
+unsafeCoerceTransform :: forall a b. Stream.Duplex -> Transform a b
-fromBufferWritable = unsafeCoerce
+unsafeCoerceTransform = unsafeCoerce
 
-fromStringReadable :: forall r. Stream.Readable r -> Readable String
+unsafeFromBufferReadable :: forall r. Stream.Readable r -> Readable Buffer
-fromStringReadable = unsafeCoerce
+unsafeFromBufferReadable = unsafeCoerce
 
-fromStringTransform :: Stream.Duplex -> Transform String String
+unsafeFromBufferTransform :: forall a. Stream.Duplex -> Transform Buffer a
-fromStringTransform = unsafeCoerce
+unsafeFromBufferTransform = unsafeCoerce
 
-fromStringWritable :: forall r. Stream.Writable r -> Writable String
+unsafeFromBufferWritable :: forall r. Stream.Writable r -> Writable Buffer
-fromStringWritable = unsafeCoerce
+unsafeFromBufferWritable = unsafeCoerce
+
+unsafeFromStringReadable :: forall r. Stream.Readable r -> Readable String
+unsafeFromStringReadable = unsafeCoerce
+
+unsafeFromStringTransform :: forall a. Stream.Duplex -> Transform String a
+unsafeFromStringTransform = unsafeCoerce
+
+unsafeFromStringWritable :: forall r. Stream.Writable r -> Writable String
+unsafeFromStringWritable = unsafeCoerce
 
 awaitReadableOrClosed :: forall s a. Read s a => s -> Aff Unit
-awaitReadableOrClosed s = do
+awaitReadableOrClosed s = Aff.supervise do
+  fiber <-
+    Aff.forkAff
+    $ parOneOf
+      [ onceAff0 readableH s $> Right unit
+      , onceAff0 closeH s $> Right unit
+      , Left <$> onceAff1 errorH s
+      ]
+  closed <- liftEffect $ isClosed s
+  readEnded <- liftEffect $ isReadableEnded s
   readable <- liftEffect $ isReadable s
   length <- liftEffect $ readableLength s
-  when (readable && length == 0)
+  if (not closed && not readEnded && readable && length == 0) then
-    $ liftEither
+    liftEither =<< Aff.joinFiber fiber
-        =<< parOneOf
+  else
-          [ onceAff0 readableH s $> Right unit
+    Aff.killFiber (error "") fiber
-          , onceAff0 closeH s $> Right unit
-          , Left <$> onceAff1 errorH s
-          ]
 
 awaitFinished :: forall s a. Write s a => s -> Aff Unit
-awaitFinished s = onceAff0 finishH s
+awaitFinished s = Aff.supervise do
+  fiber <- Aff.forkAff $ onceAff0 finishH s
+  finished <- liftEffect $ isWritableFinished s
+  if not finished then Aff.joinFiber fiber else Aff.killFiber (error "") fiber
 
 awaitWritableOrClosed :: forall s a. Write s a => s -> Aff Unit
-awaitWritableOrClosed s = do
+awaitWritableOrClosed s = Aff.supervise do
+  fiber <-
+    Aff.forkAff
+    $ parOneOf
+        [ onceAff0 drainH s $> Right unit
+        , onceAff0 closeH s $> Right unit
+        , Left <$> onceAff1 errorH s
+        ]
+  closed <- liftEffect $ isClosed s
+  writeEnded <- liftEffect $ isWritableEnded s
   writable <- liftEffect $ isWritable s
   needsDrain <- liftEffect $ needsDrain s
-  when (writable && needsDrain)
+  if not closed && not writeEnded && writable && needsDrain then
-    $ liftEither =<< parOneOf [ onceAff0 drainH s $> Right unit, onceAff0 closeH s $> Right unit, Left <$> onceAff1 errorH s ]
+    liftEither =<< Aff.joinFiber fiber
+  else
+    Aff.killFiber (error "") fiber
 
 onceAff0 :: forall e. EventHandle0 e -> e -> Aff Unit
 onceAff0 h emitter = makeAff \res -> do

src/Pipes.Async.purs

@@ -0,0 +1,281 @@
+module Pipes.Async where
+
+import Prelude hiding (join)
+
+import Control.Alternative (class Alternative, empty, guard)
+import Control.Monad.Error.Class (class MonadError, catchError, throwError)
+import Control.Monad.Except (ExceptT, runExceptT)
+import Control.Monad.Fork.Class (class MonadBracket, class MonadFork, fork)
+import Control.Monad.Maybe.Trans (runMaybeT)
+import Control.Monad.Morph (hoist)
+import Control.Monad.Rec.Class (class MonadRec, Step(..), forever, tailRecM)
+import Control.Monad.ST.Class (liftST)
+import Control.Monad.ST.Ref (STRef)
+import Control.Monad.ST.Ref as ST.Ref
+import Control.Monad.Trans.Class (lift)
+import Control.Parallel (class Parallel, parOneOf)
+import Data.Array (fold)
+import Data.Array as Array
+import Data.DateTime.Instant as Instant
+import Data.Either (Either(..), either)
+import Data.Generic.Rep (class Generic)
+import Data.Maybe (Maybe(..), isNothing)
+import Data.Newtype (unwrap)
+import Data.Show.Generic (genericShow)
+import Data.Time.Duration (Milliseconds)
+import Data.Traversable (traverse_)
+import Data.Tuple.Nested (type (/\), (/\))
+import Effect.Aff.Class (class MonadAff)
+import Effect.Class (liftEffect)
+import Effect.Console (log)
+import Effect.Exception (Error)
+import Effect.Now as Now
+import Pipes (await, yield)
+import Pipes.Collect as Collect
+import Pipes.Core (Pipe, Producer, Proxy)
+
+data WriteSignal
+  = WriteSignalOk
+  | WriteSignalEnded
+derive instance Generic WriteSignal _
+derive instance Eq WriteSignal
+derive instance Ord WriteSignal
+instance Show WriteSignal where show = genericShow
+
+instance Discard WriteSignal where
+  discard = bind
+
+data ReadSignal
+  = ReadSignalOk
+  | ReadSignalEnded
+
+derive instance Generic ReadSignal _
+derive instance Eq ReadSignal
+derive instance Ord ReadSignal
+instance Show ReadSignal where show = genericShow
+
+instance Discard ReadSignal where
+  discard = bind
+
+data WriteResult
+  = WriteAgain
+  | WriteNeedsDrain
+  | WriteEnded
+derive instance Generic WriteResult _
+derive instance Eq WriteResult
+derive instance Ord WriteResult
+instance Show WriteResult where show = genericShow
+
+data ReadResult a
+  = ReadOk a
+  | ReadWouldBlock
+derive instance Generic (ReadResult a) _
+derive instance Eq a => Eq (ReadResult a)
+derive instance Ord a => Ord (ReadResult a)
+derive instance Functor ReadResult
+instance Show a => Show (ReadResult a) where show = genericShow
+
+-- | An `AsyncPipe` is a `Pipe`-like struct that allows
+-- | concurrently reading from a `Producer` and writing to a `Consumer`.
+-- |
+-- | An implementation of `AsyncPipe` for Node `Transform` streams
+-- | is provided in `Pipes.Node.Stream`.
+-- |
+-- | ## Fields
+-- | - `m x`
+-- |    - Initializer
+-- | - `x -> a -> m WriteResult`
+-- |    - Write a value `a` to the underlying resource
+-- | - `x -> m WriteSignal`
+-- |    - Block until the pipe is writable again (or writing must stop)
+-- | - `x -> m (ReadResult b)`
+-- |    - Attempt to read a chunk
+-- | - `x -> m ReadSignal`
+-- |    - Block until the pipe is readable again (or reading must stop)
+data AsyncPipe x a b m =
+  AsyncPipe
+    (m x)
+    (x -> a -> m WriteResult)
+    (x -> m WriteSignal)
+    (x -> m (ReadResult b))
+    (x -> m ReadSignal)
+
+-- | Wraps all fields of an `AsyncPipe` with logging to debug
+-- | behavior and timing.
+debug :: forall x a b m. MonadAff m => String -> AsyncPipe x a b m -> AsyncPipe x a b m
+debug c (AsyncPipe init write awaitWrite read awaitRead) =
+  let
+    logL m = liftEffect $ log $ "[" <> c <> "] " <> m
+    logR m = liftEffect $ log $ "[" <> c <> "] " <> fold (Array.replicate 20 " ") <> m
+
+    time :: forall a'. m a' -> m (Milliseconds /\ a')
+    time m = do
+      start <- liftEffect Now.now
+      a <- m
+      end <- liftEffect Now.now
+      pure $ (end `Instant.diff` start) /\ a
+
+    init' = do
+      logL "init >"
+      elapsed /\ x <- time init
+      logL $ "< init " <> "(" <> show (unwrap elapsed) <> "ms)"
+      pure x
+
+    write' x a = do
+      logL "write >"
+      elapsed /\ w <- time $ write x a
+      logL $ "< write " <> show w <> " (" <> show (unwrap elapsed) <> "ms)"
+      pure w
+
+    read' x = do
+      logR "read >"
+      elapsed /\ r <- time $ read x
+      logR $ "< read " <> show (r $> unit) <> " (" <> show (unwrap elapsed) <> "ms)"
+      pure r
+
+    awaitWrite' x = do
+      logL "awaitWrite >"
+      elapsed /\ w <- time $ awaitWrite x
+      logL $ "< awaitWrite " <> show w <> " (" <> show (unwrap elapsed) <> "ms)"
+      pure w
+
+    awaitRead' x = do
+      logR "awaitRead >"
+      elapsed /\ r <- time $ awaitRead x
+      logR $ "< awaitRead " <> show r <> " (" <> show (unwrap elapsed) <> "ms)"
+      pure r
+  in
+    AsyncPipe init' write' awaitWrite' read' awaitRead'
+
+-- | Convert an `AsyncPipe` to a regular `Pipe`.
+-- |
+-- | Rather than two concurrently-running halves (producer & consumer),
+-- | this requires the `AsyncPipe` to occasionally stop `await`ing data
+-- | written by the upstream `Producer` so that it can `yield` to the downstream `Consumer`.
+-- |
+-- | This implementation chooses to prioritize `yield`ing data to the `Consumer` over
+-- | `await`ing written chunks.
+-- |
+-- | Note that using this limits the potential parallelism of the entire pipeline, ex:
+-- |
+-- | ```purs
+-- | Pipe.FS.read "foo.csv"        -- read
+-- |   >-> sync Pipe.CSV.parse     -- parse
+-- |   >-> sync Pipe.CBOR.encode   -- encode
+-- |   >-> Pipe.FS.write "foo.bin" -- write
+-- | ```
+-- |
+-- | In the above example, this is what happens when the pipeline
+-- | is executed:
+-- | 1. `write` asks `encode` "do you have any data yet?" (fast)
+-- | 1. `encode` asks `parse` "do you have any data yet?" (fast)
+-- | 1. `parse` asks `read` "do you have any data yet?" (fast)
+-- | 1. `read` passes 1 chunk to `parse` (fast)
+-- | 1. `parse` blocks until the chunk is parsed (slow)
+-- | 1. `parse` passes 1 chunk to `encode` (fast)
+-- | 1. `encode` blocks until the chunk is encoded (slow)
+-- | 1. `write` writes the block (fast)
+-- |
+-- | For larger workloads, changing this to use `asyncPipe` would be preferable, ex:
+-- | ```purs
+-- | Pipe.FS.read "foo.csv"        -- read
+-- |   >-/-> Pipe.CSV.parse        -- parse
+-- |   >-/-> Pipe.CBOR.encode      -- encode
+-- |   >-> Pipe.FS.write "foo.bin" -- write
+-- | ```
+-- |
+-- | With this change:
+-- | * `read` will pass chunks to `parse` as fast as `parse` allows
+-- | * `parse` will parse chunks and yield them to `encode` as soon as they're ready
+-- | * `encode` will encode chunks and yield them to `write` as soon as they're ready
+sync :: forall x a b f p e m. MonadError e m => Alternative p => Parallel p m => MonadFork f m => MonadAff m => AsyncPipe x (Maybe a) (Maybe b) m -> Pipe (Maybe a) (Maybe b) m Unit
+sync (AsyncPipe init write awaitWrite read awaitRead) =
+  let
+    liftPipe :: forall r. (Proxy _ _ _ _ m) r -> ExceptT (Step _ _) (Proxy _ _ _ _ m) r
+    liftPipe = lift
+
+    liftM :: forall r. m r -> ExceptT (Step _ _) (Proxy _ _ _ _ m) r
+    liftM = liftPipe <<< lift
+
+    continue a = throwError (Loop a)
+    break = throwError (Done unit)
+
+    awaitRW x = parOneOf [Right <$> awaitWrite x, Left <$> awaitRead x]
+
+    wSignal WriteSignalOk = WriteAgain
+    wSignal WriteSignalEnded = WriteEnded
+  in do
+    x <- lift init
+    flip tailRecM WriteAgain
+      \w ->
+      map (either identity identity)
+      $ runExceptT do
+          rb <- liftM $ read x
+          case rb of
+            ReadWouldBlock
+              | w == WriteEnded -> liftM (awaitRead x) *> continue w
+              | w == WriteNeedsDrain -> liftM (awaitRW x) >>= either (const $ continue w) (continue <<< wSignal)
+              | otherwise -> pure unit
+            ReadOk (Just b) -> liftPipe (yield $ Just b) *> continue w
+            ReadOk Nothing -> liftPipe (yield Nothing) *> break
+
+          when (w /= WriteAgain) $ continue w
+
+          a <- liftPipe await
+          w' <- liftM $ write x a
+          when (isNothing a) $ continue WriteEnded
+          pure $ Loop w'
+
+-- | Implementation of `(>-/->)`
+-- |
+-- | In the current `MonadFork` "thread", read data from the `AsyncPipe` as it
+-- | is yielded and `yield` to the downstream `Consumer`.
+-- |
+-- | Concurrently, in a separate thread, read data from the upstream `Producer`
+-- | and write to the `AsyncPipe` at max throughput.
+-- |
+-- | If the producing half fails, the error is caught and rethrown.
+-- |
+-- | If the consuming half fails, the error is caught, the producing half is killed, and the error is rethrown.
+pipeAsync
+  :: forall f m x a b
+   . MonadRec m
+  => MonadAff m
+  => MonadBracket Error f m
+  => Producer (Maybe a) m Unit
+  -> AsyncPipe x (Maybe a) (Maybe b) m
+  -> Producer (Maybe b) m Unit
+pipeAsync prod (AsyncPipe init write awaitWrite read awaitRead) =
+  do
+    errST :: STRef _ (Maybe Error) <- lift $ liftEffect $ liftST $ ST.Ref.new Nothing
+    killST :: STRef _ Boolean <- lift $ liftEffect $ liftST $ ST.Ref.new false
+
+    let
+      killThread = void $ liftEffect $ liftST $ ST.Ref.write true killST
+      threadKilled = liftEffect $ liftST $ ST.Ref.read killST
+      putThreadError = void <<< liftEffect <<< liftST <<< flip ST.Ref.write errST <<< Just
+      getThreadError = liftEffect $ liftST $ ST.Ref.read errST
+
+      rx x a = do
+        killed <- threadKilled
+        guard $ not killed
+        w <- lift $ write x a
+        case w of
+          WriteNeedsDrain -> lift $ void $ awaitWrite x
+          WriteEnded -> empty
+          WriteAgain -> pure unit
+
+      spawn = lift <<< fork <<< flip catchError putThreadError
+
+    x <- lift init
+    _thread <- spawn $ void $ runMaybeT $ Collect.foreach (rx x) (hoist lift prod)
+
+    forever do
+      getThreadError >>= traverse_ throwError
+      rb <- lift $ read x
+      case rb of
+        ReadOk (Just b) -> yield $ Just b
+        ReadOk Nothing -> killThread *> yield Nothing
+        ReadWouldBlock -> void $ lift (awaitRead x)
+
+infixl 7 pipeAsync as >-/->

src/Pipes.Collect.purs

@@ -20,6 +20,7 @@ import Foreign.Object.ST as Object.ST
 import Foreign.Object.ST.Unsafe as Object.ST.Unsafe
 import Node.Buffer (Buffer)
 import Node.Buffer as Buffer
+import Node.Encoding (Encoding)
 import Pipes.Core (Producer)
 import Pipes.Internal (Proxy(..))
 
@@ -30,12 +31,8 @@ traverse :: forall a b m. MonadRec m => (b -> a -> m b) -> b -> Producer a m Uni
 traverse f b0 p0 =
   flip tailRecM (p0 /\ b0) \(p /\ b) ->
     case p of
-      Respond a m -> do
+      Respond a m -> Loop <$> (m unit /\ _) <$> f b a
-        b' <- f b a
+      M m -> Loop <$> (_ /\ b) <$> m
-        pure $ Loop $ m unit /\ b'
-      M m -> do
-        n <- m
-        pure $ Loop $ (n /\ b)
       Request _ _ -> pure $ Done b
       Pure _ -> pure $ Done b
 
@@ -51,7 +48,16 @@ fold f b0 p0 = traverse (\b a -> pure $ f b a) b0 p0
 foreach :: forall a m. MonadRec m => (a -> m Unit) -> Producer a m Unit -> m Unit
 foreach f p0 = traverse (\_ a -> f a) unit p0
 
--- | Concatenate all produced buffers
+-- | `append` all emitted values to `mempty`
+toMonoid :: forall a m. Monoid a => MonadRec m => MonadEffect m => Producer a m Unit -> m a
+toMonoid = fold (<>) mempty
+
+-- | Concatenate all buffers to a single buffer, then decode with the
+-- | provided encoding.
+toStringWith :: forall m. MonadRec m => MonadEffect m => Encoding -> Producer Buffer m Unit -> m String
+toStringWith enc = (liftEffect <<< Buffer.toString enc) <=< toBuffer
+
+-- | Concatenate all produced buffers to a single buffer
 toBuffer :: forall m. MonadRec m => MonadEffect m => Producer Buffer m Unit -> m Buffer
 toBuffer p =
   (liftEffect <<< maybe (Buffer.alloc 0) pure)
@@ -74,9 +80,18 @@ toArray p = do
   liftEffect $ liftST $ Array.ST.unsafeFreeze st
 
 -- | Collect all values from a `Producer` into a list.
+-- |
+-- | Reverses the list after collecting, so that values will be
+-- | in the order they were emitted.
 toList :: forall a m. MonadRec m => MonadEffect m => Producer a m Unit -> m (List a)
 toList = map List.reverse <<< fold (flip List.Cons) List.Nil
 
+-- | Collect all values from a `Producer` into a list.
+-- |
+-- | Does not reverse the list after collecting.
+toListRev :: forall a m. MonadRec m => MonadEffect m => Producer a m Unit -> m (List a)
+toListRev = map List.reverse <<< fold (flip List.Cons) List.Nil
+
 -- | Collect all values from a `Producer` into a Javascript Object.
 toObject :: forall a m. MonadRec m => MonadEffect m => Producer (String /\ a) m Unit -> m (Object a)
 toObject p = do

src/Pipes.Node.FS.purs

@@ -23,7 +23,7 @@ import Prim.Row (class Union)
 -- |
 -- | See `Pipes.Node.Stream.withEOS` for converting `Producer a`
 -- | into `Producer (Maybe a)`, emitting `Nothing` before exiting.
-write
+write'
   :: forall r trash m
    . Union r trash WriteStreamOptions
   => MonadAff m
@@ -31,27 +31,27 @@ write
   => Record r
   -> FilePath
   -> Consumer (Maybe Buffer) m Unit
-write o p = do
+write' o p = do
   w <- liftEffect $ FS.Stream.createWriteStream' p o
-  fromWritable $ O.fromBufferWritable w
+  fromWritable $ O.unsafeCoerceWritable w
 
 -- | Open a file in write mode, failing if the file already exists.
 -- |
--- | `write {flags: "wx"}`
+-- | `write' {flags: "wx"}`
 create :: forall m. MonadAff m => MonadThrow Error m => FilePath -> Consumer (Maybe Buffer) m Unit
-create = write { flags: "wx" }
+create = write' { flags: "wx" }
 
 -- | Open a file in write mode, truncating it if the file already exists.
 -- |
--- | `write {flags: "w"}`
+-- | `write' {flags: "w"}`
-truncate :: forall m. MonadAff m => MonadThrow Error m => FilePath -> Consumer (Maybe Buffer) m Unit
+trunc :: forall m. MonadAff m => MonadThrow Error m => FilePath -> Consumer (Maybe Buffer) m Unit
-truncate = write { flags: "w" }
+trunc = write' { flags: "w" }
 
 -- | Open a file in write mode, appending written contents if the file already exists.
 -- |
--- | `write {flags: "a"}`
+-- | `write' {flags: "a"}`
 append :: forall m. MonadAff m => MonadThrow Error m => FilePath -> Consumer (Maybe Buffer) m Unit
-append = write { flags: "a" }
+append = write' { flags: "a" }
 
 -- | Creates a `fs.Readable` stream for the file at the given path.
 -- |
@@ -60,7 +60,7 @@ append = write { flags: "a" }
 read :: forall m. MonadAff m => MonadThrow Error m => FilePath -> Producer (Maybe Buffer) m Unit
 read p = do
   r <- liftEffect $ FS.Stream.createReadStream p
-  fromReadable $ O.fromBufferReadable r
+  fromReadable $ O.unsafeCoerceReadable r
 
 -- | Creates a `fs.Readable` stream for the file at the given path.
 -- |
@@ -76,4 +76,4 @@ read'
   -> Producer (Maybe Buffer) m Unit
 read' opts p = do
   r <- liftEffect $ FS.Stream.createReadStream' p opts
-  fromReadable $ O.fromBufferReadable r
+  fromReadable $ O.unsafeCoerceReadable r

src/Pipes.Node.Stream.purs

@@ -5,21 +5,25 @@ import Prelude hiding (join)
 import Control.Monad.Error.Class (class MonadThrow, throwError)
 import Control.Monad.Rec.Class (class MonadRec, Step(..), tailRecM)
 import Control.Monad.ST.Class (liftST)
+import Control.Monad.ST.Global (Global)
+import Control.Monad.ST.Ref (STRef)
+import Control.Monad.ST.Ref as ST.Ref
 import Control.Monad.ST.Ref as STRef
 import Control.Monad.Trans.Class (lift)
-import Data.Maybe (Maybe(..))
+import Data.Maybe (Maybe(..), maybe)
-import Data.Newtype (wrap)
+import Data.Traversable (for_, traverse_)
-import Data.Traversable (for_)
 import Data.Tuple.Nested ((/\))
-import Effect.Aff (delay)
+import Effect (Effect)
 import Effect.Aff.Class (class MonadAff, liftAff)
-import Effect.Class (liftEffect)
+import Effect.Class (class MonadEffect, liftEffect)
 import Effect.Exception (Error)
+import Node.Stream.Object (WriteResult(..), maybeReadResult)
 import Node.Stream.Object as O
 import Pipes (await, yield)
 import Pipes (for) as P
+import Pipes.Async (AsyncPipe(..))
+import Pipes.Async as AsyncPipe
 import Pipes.Core (Consumer, Pipe, Producer, Producer_)
-import Pipes.Prelude (mapFoldable) as P
 import Pipes.Util (InvokeResult(..), invoke)
 
 -- | Convert a `Readable` stream to a `Pipe`.
@@ -57,90 +61,107 @@ fromReadable r =
 -- | When `Nothing` is piped to this, the stream will
 -- | be `end`ed, and the pipe will noop if invoked again.
 fromWritable :: forall s a m. MonadThrow Error m => MonadAff m => O.Write s a => s -> Consumer (Maybe a) m Unit
-fromWritable w =
+fromWritable w = do
+  { error: errorST, cancel: removeErrorListener } <- liftEffect $ O.withErrorST w
+
   let
-    cleanup rmErrorListener = do
+    maybeThrow = traverse_ throwError =<< liftEffect (liftST $ STRef.read errorST)
-      liftEffect rmErrorListener
+
-      liftEffect $ O.end w
+    waitCanWrite = do
+      shouldWait <- liftEffect $ O.needsDrain w
+      when shouldWait $ liftAff $ O.awaitWritableOrClosed w
+
+    cleanup = do
       liftAff $ O.awaitFinished w
-      pure $ Done unit
+      maybeThrow
+      liftEffect removeErrorListener
 
-    go { error, cancel } = do
+    onEOS = liftEffect (O.end w) *> cleanup $> Done unit
-      err <- liftEffect $ liftST $ STRef.read error
+    onChunk a = liftEffect (O.write w a) $> Loop unit
-      for_ err throwError
-
-      needsDrain <- liftEffect $ O.needsDrain w
-      when needsDrain $ liftAff $ O.awaitWritableOrClosed w
 
+    go _ = do
+      maybeThrow
+      waitCanWrite
       ended <- liftEffect $ O.isWritableEnded w
       if ended then
-        cleanup cancel
+        cleanup $> Done unit
       else
-        await >>= case _ of
+        await >>= maybe onEOS onChunk
-          Nothing -> cleanup cancel
-          Just a -> do
-            void $ liftEffect $ O.write w a
-            pure $ Loop { error, cancel }
-  in
-    do
-      r <- liftEffect $ O.withErrorST w
-      tailRecM go r
 
--- | Convert a `Transform` stream to a `Pipe`.
+  tailRecM go unit
--- |
+
--- | When `Nothing` is piped to this, the `Transform` stream will
+newtype TransformContext a b =
--- | be `end`ed, and the pipe will noop if invoked again.
+  TransformContext
-fromTransform :: forall a b m. MonadThrow Error m => MonadAff m => O.Transform a b -> Pipe (Maybe a) (Maybe b) m Unit
+    { stream :: O.Transform a b
+    , removeErrorListener :: Effect Unit
+    , errorST :: STRef Global (Maybe Error)
+    }
+
+transformCleanup :: forall m a b. MonadEffect m => TransformContext a b -> m Unit
+transformCleanup (TransformContext {removeErrorListener}) = do
+  liftEffect removeErrorListener
+
+transformStream :: forall a b. TransformContext a b -> O.Transform a b
+transformStream (TransformContext {stream}) = stream
+
+transformRethrow :: forall m a b. MonadThrow Error m => MonadEffect m => TransformContext a b -> m Unit
+transformRethrow (TransformContext {errorST}) = traverse_ throwError =<< liftEffect (liftST $ ST.Ref.read errorST)
+
+-- | Convert a `Transform` stream to an `AsyncPipe`.
+fromTransform
+  :: forall a b m
+   . MonadThrow Error m
+  => MonadAff m
+  => Effect (O.Transform a b)
+  -> AsyncPipe (TransformContext a b) (Maybe a) (Maybe b) m
 fromTransform t =
   let
-    cleanup removeErrorListener = do
+    init = do
-      liftEffect $ O.end t
+      stream <- liftEffect t
-      liftAff $ O.awaitFinished t
+      { error: errorST, cancel: removeErrorListener } <- liftEffect $ O.withErrorST stream
-      fromReadable t
+      pure $ TransformContext {errorST, removeErrorListener, stream}
-      liftEffect $ removeErrorListener
+    write x Nothing = do
-      pure $ Done unit
+      let s = transformStream x
-
+      liftEffect $ O.end s
-    yieldWhileReadable = do
+      pure AsyncPipe.WriteEnded
-      flip tailRecM unit \_ -> do
+    write x (Just a) = do
-        res <- liftEffect $ O.read t
+      transformRethrow x
-        case res of
+      let s = transformStream x
-          O.ReadJust a -> yield (Just a) $> Loop unit
+      w <- liftEffect $ O.write s a
-          _ -> pure $ Done unit
+      pure $ case w of
-
+        WriteOk -> AsyncPipe.WriteAgain
-    maybeYield1 = do
+        WriteWouldBlock -> AsyncPipe.WriteNeedsDrain
-      res <- liftEffect $ O.read t
+    awaitWrite x = do
-      case res of
+      transformRethrow x
-        O.ReadJust a -> yield $ Just a
+      let s = transformStream x
-        _ -> pure unit
+      liftAff $ O.awaitWritableOrClosed s
-
+      ended <- liftEffect $ O.isWritableEnded s
-    go { error, cancel } = do
+      if ended then
-      err <- liftEffect $ liftST $ STRef.read error
+        pure $ AsyncPipe.WriteSignalEnded
-      for_ err throwError
+      else do
-
+        liftAff $ O.awaitWritableOrClosed s
-      needsDrain <- liftEffect $ O.needsDrain t
+        pure $ AsyncPipe.WriteSignalOk
-      ended <- liftEffect $ O.isWritableEnded t
+    read x =
-      if needsDrain then do
+      do
-        liftAff $ delay $ wrap 0.0
+        transformRethrow x
-        yieldWhileReadable
+        let s = transformStream x
-        pure $ Loop { error, cancel }
+        readEnded <- liftEffect $ O.isReadableEnded s
-      else if ended then
+        if readEnded then do
-        cleanup cancel
+          transformCleanup x
-      else
+          pure $ AsyncPipe.ReadOk Nothing
-        await >>= case _ of
+        else
-          Nothing -> cleanup cancel
+          maybe AsyncPipe.ReadWouldBlock (AsyncPipe.ReadOk <<< Just) <$> maybeReadResult <$> liftEffect (O.read s)
-          Just a' -> do
+    awaitRead x = do
-            res <- liftEffect $ O.write t a'
+      transformRethrow x
-            case res of
+      let s = transformStream x
-              O.WriteOk -> do
+      ended <- liftEffect $ O.isReadableEnded s
-                maybeYield1
+      if ended then
-                pure $ Loop { error, cancel }
+        pure $ AsyncPipe.ReadSignalEnded
-              O.WriteWouldBlock -> do
+      else do
-                yieldWhileReadable
+        liftAff $ O.awaitReadableOrClosed s
-                pure $ Loop { error, cancel }
+        pure $ AsyncPipe.ReadSignalOk
   in
-    do
+    AsyncPipe init write awaitWrite read awaitRead
-      r <- liftEffect $ O.withErrorST t
-      tailRecM go r
 
 -- | Given a `Producer` of values, wrap them in `Just`.
 -- |
@@ -152,7 +173,7 @@ withEOS a = do
 
 -- | Strip a pipeline of the EOS signal
 unEOS :: forall a m. Monad m => Pipe (Maybe a) a m Unit
-unEOS = P.mapFoldable identity
+unEOS = tailRecM (const $ maybe (pure $ Done unit) (\a -> yield a $> Loop unit) =<< await) unit
 
 -- | Lift a `Pipe a a` to `Pipe (Maybe a) (Maybe a)`.
 -- |

src/Pipes.Node.Zlib.purs

@@ -12,31 +12,34 @@ import Node.Buffer (Buffer)
 import Node.Stream.Object as O
 import Node.Zlib as Zlib
 import Node.Zlib.Types (ZlibStream)
-import Pipes.Core (Pipe)
+import Pipes.Async (AsyncPipe)
-import Pipes.Node.Stream (fromTransform)
+import Pipes.Node.Stream (TransformContext, fromTransform)
 
-fromZlib :: forall r m. MonadAff m => MonadThrow Error m => Effect (ZlibStream r) -> Pipe (Maybe Buffer) (Maybe Buffer) m Unit
+type X = TransformContext Buffer Buffer
-fromZlib z = do
-  raw <- liftEffect $ Zlib.toDuplex <$> z
-  fromTransform $ O.fromBufferTransform raw
 
-gzip :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
+fromZlib :: forall r m. MonadAff m => MonadThrow Error m => Effect (ZlibStream r) -> AsyncPipe X (Maybe Buffer) (Maybe Buffer) m
+fromZlib z =
+  fromTransform do
+    raw <- liftEffect $ Zlib.toDuplex <$> z
+    pure $ O.unsafeCoerceTransform raw
+
+gzip :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe X (Maybe Buffer) (Maybe Buffer) m
 gzip = fromZlib Zlib.createGzip
 
-gunzip :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
+gunzip :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe X (Maybe Buffer) (Maybe Buffer) m
 gunzip = fromZlib Zlib.createGunzip
 
-unzip :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
+unzip :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe X (Maybe Buffer) (Maybe Buffer) m
 unzip = fromZlib Zlib.createUnzip
 
-inflate :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
+inflate :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe X (Maybe Buffer) (Maybe Buffer) m
 inflate = fromZlib Zlib.createInflate
 
-deflate :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
+deflate :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe X (Maybe Buffer) (Maybe Buffer) m
 deflate = fromZlib Zlib.createDeflate
 
-brotliCompress :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
+brotliCompress :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe X (Maybe Buffer) (Maybe Buffer) m
 brotliCompress = fromZlib Zlib.createBrotliCompress
 
-brotliDecompress :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
+brotliDecompress :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe X (Maybe Buffer) (Maybe Buffer) m
 brotliDecompress = fromZlib Zlib.createBrotliDecompress

src/Pipes.Util.purs

@@ -86,9 +86,14 @@ chunked size = do
     a <- MaybeT await
     chunkPut a
     len <- lift chunkLength
-    when (len >= size) $ lift $ yield =<< Just <$> chunkTake
+    when (len >= size) do
+      chunk <- lift chunkTake
+      lift $ yield $ Just chunk
   len <- chunkLength
-  when (len > 0) $ yield =<< Just <$> chunkTake
+  when (len > 0) do
+    chunk <- chunkTake
+    yield $ Just chunk
+
   yield Nothing
 
 -- | Equivalent of unix `uniq`, filtering out duplicate values passed to it.
@@ -144,3 +149,5 @@ invoke m a =
         Pure _ -> pure Exited
   in
     go (IDidNotAwait m)
+
+

test/Test/Main.purs

@@ -3,18 +3,19 @@ module Test.Main where
 import Prelude
 
 import Data.Maybe (Maybe(..))
+import Data.Time.Duration (Milliseconds(..))
 import Effect (Effect)
 import Effect.Aff (launchAff_)
-import Test.Pipes.Node.Stream as Test.Pipes.Node.Stream
-import Test.Pipes.Node.Buffer as Test.Pipes.Node.Buffer
-import Test.Pipes.Node.FS as Test.Pipes.Node.FS
 import Test.Pipes.Collect as Test.Pipes.Collect
 import Test.Pipes.Construct as Test.Pipes.Construct
+import Test.Pipes.Node.Buffer as Test.Pipes.Node.Buffer
+import Test.Pipes.Node.FS as Test.Pipes.Node.FS
+import Test.Pipes.Node.Stream as Test.Pipes.Node.Stream
 import Test.Spec.Reporter (specReporter)
 import Test.Spec.Runner (defaultConfig, runSpec')
 
 main :: Effect Unit
-main = launchAff_ $ runSpec' (defaultConfig { exit = false, timeout = Nothing }) [ specReporter ] do
+main = launchAff_ $ runSpec' (defaultConfig { slow = Milliseconds 0.0, failFast = true, exit = false, timeout = Nothing }) [ specReporter ] do
   Test.Pipes.Node.Stream.spec
   Test.Pipes.Node.Buffer.spec
   Test.Pipes.Node.FS.spec

test/Test/Pipes.Node.FS.purs

@@ -33,11 +33,6 @@ spec = describe "Pipes.Node.FS" do
       liftEffect $ FS.writeTextFile UTF8 p "foo"
       s <- fold <$> Pipes.toListM (Pipes.Node.FS.read p >-> unEOS >-> Pipes.Node.Buffer.toString UTF8)
       s `shouldEqual` "foo"
-    around tmpFile $ it "fails if the file already exists" \p -> do
-      liftEffect $ FS.writeTextFile UTF8 p "foo"
-      flip catchError (const $ pure unit) do
-        Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.create p
-        fail "should have thrown"
   describe "create" do
     around tmpFile $ it "creates the file when not exists" \p -> do
       Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.create p
@@ -59,14 +54,14 @@ spec = describe "Pipes.Node.FS" do
       Pipes.runEffect $ withEOS (yield "bar" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.append p
       contents <- liftEffect $ FS.readTextFile UTF8 p
       contents `shouldEqual` "foo\nbar"
-  describe "truncate" do
+  describe "trunc" do
     around tmpFile $ it "creates the file when not exists" \p -> do
-      Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.truncate p
+      Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.trunc p
       contents <- liftEffect $ FS.readTextFile UTF8 p
       contents `shouldEqual` "foo"
     around tmpFile $ it "overwrites contents" \p -> do
-      Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.truncate p
+      Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.trunc p
-      Pipes.runEffect $ withEOS (yield "bar" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.truncate p
+      Pipes.runEffect $ withEOS (yield "bar" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.trunc p
       contents <- liftEffect $ FS.readTextFile UTF8 p
       contents `shouldEqual` "bar"
   around tmpFiles $ it "json lines >-> parse >-> _.foo >-> write" \(a /\ b) -> do

test/Test/Pipes.Node.Stream.js

@@ -1,4 +1,16 @@
 import Stream from 'stream'
+import * as CBOR from "cbor-x";
+import * as CSVDecode from "csv-parse";
+import * as CSVEncode from "csv-stringify";
+
+export const cborDecode = () => new CBOR.DecoderStream({useRecords: false, allowHalfOpen: true});
+export const cborEncode = () => new CBOR.EncoderStream({useRecords: false, allowHalfOpen: true});
+
+export const cborDecodeSync = a => () => CBOR.decodeMultiple(a);
+export const cborEncodeSync = a => () => CBOR.encode(a, {useRecords: false});
+
+export const csvDecode = () => CSVDecode.parse({columns: true, allowHalfOpen: true})
+export const csvEncode = () => CSVEncode.stringify({header: true, allowHalfOpen: true})
 
 export const discardTransform = () => new Stream.Transform({
   transform: function(_ck, _enc, cb) {
@@ -7,6 +19,16 @@ export const discardTransform = () => new Stream.Transform({
   objectMode: true
 })
 
+export const slowTransform = () => {
+  return new Stream.Transform({
+  transform: function(ck, _enc, cb) {
+    this.push(ck)
+    setTimeout(() => cb(), 4)
+  },
+  objectMode: true
+})
+}
+
 export const charsTransform = () => new Stream.Transform({
   transform: function(ck, _enc, cb) {
     ck.split('').filter(s => !!s).forEach(s => this.push(s))

test/Test/Pipes.Node.Stream.purs

@@ -2,49 +2,72 @@ module Test.Pipes.Node.Stream where
 
 import Prelude
 
-import Control.Monad.Trans.Class (lift)
+import Control.Monad.Cont (lift)
+import Control.Monad.Error.Class (liftEither)
+import Control.Monad.Except (runExcept)
 import Data.Array as Array
-import Data.Foldable (fold)
+import Data.Bifunctor (lmap)
+import Data.Either (Either(..))
+import Data.Foldable (fold, intercalate)
+import Data.FoldableWithIndex (forWithIndex_)
+import Data.FunctorWithIndex (mapWithIndex)
+import Data.Int as Int
 import Data.List ((:))
 import Data.List as List
 import Data.Maybe (Maybe)
 import Data.Newtype (wrap)
+import Data.Profunctor.Strong (first)
+import Data.String as String
 import Data.String.Gen (genAlphaString)
+import Data.Traversable (for_, traverse)
 import Data.Tuple.Nested (type (/\), (/\))
 import Effect (Effect)
 import Effect.Aff (Aff, delay)
 import Effect.Class (class MonadEffect, liftEffect)
+import Effect.Exception (error)
+import Effect.Unsafe (unsafePerformEffect)
+import Foreign (Foreign)
 import Node.Buffer (Buffer)
 import Node.Buffer as Buffer
 import Node.Encoding (Encoding(..))
 import Node.FS.Stream as FS.Stream
 import Node.FS.Sync as FS
 import Node.Stream.Object as O
-import Node.Zlib as Zlib
+import Pipes (each) as Pipe
-import Pipes (each) as Pipes
 import Pipes (yield, (>->))
+import Pipes.Async (sync, (>-/->))
+import Pipes.Collect as Pipe.Collect
 import Pipes.Core (Consumer, Producer, runEffect)
-import Pipes.Node.Buffer as Pipes.Buffer
+import Pipes.Node.Buffer as Pipe.Buffer
-import Pipes.Node.Stream as S
+import Pipes.Node.FS as Pipe.FS
-import Pipes.Prelude (mapFoldable, toListM) as Pipes
+import Pipes.Node.Stream as Pipe.Node
-import Simple.JSON (writeJSON)
+import Pipes.Node.Zlib as Pipe.Zlib
-import Test.Common (jsonParse, jsonStringify, tmpFile, tmpFiles)
+import Pipes.Prelude (toListM) as Pipe
+import Simple.JSON (readImpl, readJSON, writeJSON)
+import Test.Common (jsonStringify, tmpFile, tmpFiles)
 import Test.QuickCheck.Arbitrary (arbitrary)
 import Test.QuickCheck.Gen (randomSample')
 import Test.Spec (Spec, around, describe, it)
-import Test.Spec.Assertions (shouldEqual)
+import Test.Spec.Assertions (fail, shouldEqual)
 
 foreign import readableFromArray :: forall @a. Array a -> O.Readable a
 foreign import discardTransform :: forall a b. Effect (O.Transform a b)
+foreign import slowTransform :: forall a b. Effect (O.Transform a b)
 foreign import charsTransform :: Effect (O.Transform String String)
+foreign import cborEncodeSync :: forall a. a -> Effect Buffer
+foreign import cborDecodeSync :: forall a. Buffer -> Effect a
+foreign import cborEncode :: forall a. Effect (O.Transform a Buffer)
+foreign import cborDecode :: forall a. Effect (O.Transform Buffer a)
+foreign import csvEncode :: forall a. Effect (O.Transform a String)
+foreign import csvDecode :: forall a. Effect (O.Transform String a)
 
 writer :: forall m. MonadEffect m => String -> m (O.Writable Buffer /\ Consumer (Maybe Buffer) Aff Unit)
 writer a = do
-  stream <- liftEffect $ O.fromBufferWritable <$> FS.Stream.createWriteStream a
+  stream <- liftEffect $ O.unsafeCoerceWritable <$> FS.Stream.createWriteStream a
-  pure $ stream /\ S.fromWritable stream
+  pure $ stream /\ Pipe.Node.fromWritable stream
 
 reader :: forall m. MonadEffect m => String -> m (Producer (Maybe Buffer) Aff Unit)
-reader a = liftEffect $ S.fromReadable <$> O.fromBufferReadable <$> FS.Stream.createReadStream a
+reader a = liftEffect $ Pipe.Node.fromReadable <$> O.unsafeCoerceReadable <$> FS.Stream.createReadStream a
 
 spec :: Spec Unit
 spec =
@@ -52,30 +75,30 @@ spec =
     describe "Readable" do
       describe "Readable.from(<Iterable>)" do
         it "empty" do
-          vals <- Pipes.toListM $ (S.fromReadable $ readableFromArray @{ foo :: String } []) >-> S.unEOS
+          vals <- Pipe.toListM $ (Pipe.Node.fromReadable $ readableFromArray @{ foo :: String } []) >-> Pipe.Node.unEOS
           vals `shouldEqual` List.Nil
         it "singleton" do
-          vals <- Pipes.toListM $ (S.fromReadable $ readableFromArray @{ foo :: String } [ { foo: "1" } ]) >-> S.unEOS
+          vals <- Pipe.toListM $ (Pipe.Node.fromReadable $ readableFromArray @{ foo :: String } [ { foo: "1" } ]) >-> Pipe.Node.unEOS
           vals `shouldEqual` ({ foo: "1" } : List.Nil)
         it "many elements" do
           let exp = (\n -> { foo: show n }) <$> Array.range 0 100
-          vals <- Pipes.toListM $ (S.fromReadable $ readableFromArray exp) >-> S.unEOS
+          vals <- Pipe.toListM $ (Pipe.Node.fromReadable $ readableFromArray exp) >-> Pipe.Node.unEOS
           vals `shouldEqual` (List.fromFoldable exp)
     describe "Writable" $ around tmpFile do
       describe "fs.WriteStream" do
         it "pipe to file" \p -> do
-          stream <- O.fromBufferWritable <$> liftEffect (FS.Stream.createWriteStream p)
+          stream <- O.unsafeCoerceWritable <$> liftEffect (FS.Stream.createWriteStream p)
           let
-            w = S.fromWritable stream
+            w = Pipe.Node.fromWritable stream
             source = do
               buf <- liftEffect $ Buffer.fromString "hello" UTF8
               yield buf
-          runEffect $ S.withEOS source >-> w
+          runEffect $ Pipe.Node.withEOS source >-> w
           contents <- liftEffect $ FS.readTextFile UTF8 p
           contents `shouldEqual` "hello"
           shouldEqual true =<< liftEffect (O.isWritableEnded stream)
         it "async pipe to file" \p -> do
-          w <- S.fromWritable <$> O.fromBufferWritable <$> liftEffect (FS.Stream.createWriteStream p)
+          w <- Pipe.Node.fromWritable <$> O.unsafeCoerceWritable <$> liftEffect (FS.Stream.createWriteStream p)
           let
             source = do
               yield "hello, "
@@ -87,7 +110,7 @@ spec =
               yield "this is a "
               lift $ delay $ wrap 5.0
               yield "test."
-          runEffect $ S.withEOS (source >-> Pipes.Buffer.fromString UTF8) >-> w
+          runEffect $ Pipe.Node.withEOS (source >-> Pipe.Buffer.fromString UTF8) >-> w
           contents <- liftEffect $ FS.readTextFile UTF8 p
           contents `shouldEqual` "hello, world! this is a test."
         it "chained pipes" \p -> do
@@ -101,40 +124,96 @@ spec =
           let
             exp = fold (writeJSON <$> objs)
           stream /\ w <- liftEffect $ writer p
-          runEffect $ S.withEOS (Pipes.each objs >-> jsonStringify >-> Pipes.Buffer.fromString UTF8) >-> w
+          runEffect $ Pipe.Node.withEOS (Pipe.each objs >-> jsonStringify >-> Pipe.Buffer.fromString UTF8) >-> w
           contents <- liftEffect $ FS.readTextFile UTF8 p
           contents `shouldEqual` exp
           shouldEqual true =<< liftEffect (O.isWritableEnded stream)
     describe "Transform" do
-      it "gzip" do
+      let
-        let
+        bignums = Array.range 1 1000
-          json = yield $ writeJSON { foo: "bar" }
+        firstNames = String.split (wrap "\n") $ unsafePerformEffect (FS.readTextFile UTF8 "./test/Test/first_names.txt")
-          exp = "1f8b0800000000000003ab564acbcf57b2524a4a2c52aa0500eff52bfe0d000000"
+        lastNames = String.split (wrap "\n") $ unsafePerformEffect (FS.readTextFile UTF8 "./test/Test/last_names.txt")
-        gzip <- S.fromTransform <$> O.fromBufferTransform <$> liftEffect (Zlib.toDuplex <$> Zlib.createGzip)
+        names n = do
-        outs :: List.List String <- Pipes.toListM (S.withEOS (json >-> Pipes.Buffer.fromString UTF8) >-> gzip >-> S.unEOS >-> Pipes.Buffer.toString Hex)
+          first <- firstNames
-        fold outs `shouldEqual` exp
+          last <- Array.take (Int.round $ Int.toNumber n / Int.toNumber (Array.length firstNames)) lastNames
-      around tmpFiles
+          pure $ first <> " " <> last
-        $ it "file >-> gzip >-> file >-> gunzip" \(a /\ b) -> do
+        people n = mapWithIndex (\ix name -> {id: show $ ix + 1, name}) (names n)
-            liftEffect $ FS.writeTextFile UTF8 a $ writeJSON [ 1, 2, 3, 4 ]
+        peopleCSV n = "id,name\n" <> intercalate "\n" ((\{id, name} -> id <> "," <> name) <$> people n)
-            areader <- liftEffect $ reader a
-            bwritestream /\ bwriter <- liftEffect $ writer b
-            gzip <- S.fromTransform <$> O.fromBufferTransform <$> liftEffect (Zlib.toDuplex <$> Zlib.createGzip)
-            runEffect $ areader >-> gzip >-> bwriter
-            shouldEqual true =<< liftEffect (O.isWritableEnded bwritestream)
 
-            gunzip <- S.fromTransform <$> O.fromBufferTransform <$> liftEffect (Zlib.toDuplex <$> Zlib.createGunzip)
+      for_ [4000, 8000, 32000, 64000, 200000] \n -> do
-            breader <- liftEffect $ reader b
+        let
-            nums <- Pipes.toListM (breader >-> gunzip >-> S.unEOS >-> Pipes.Buffer.toString UTF8 >-> jsonParse @(Array Int) >-> Pipes.mapFoldable identity)
+          csv = peopleCSV n
-            Array.fromFoldable nums `shouldEqual` [ 1, 2, 3, 4 ]
+          people' = people n
+        around tmpFiles
+          $ it (show n <> " row csv >-/-> csv-parse >-/-> cborEncode") \(a /\ _) -> do
+              liftEffect $ FS.writeTextFile UTF8 a csv
+              cbor :: Buffer <- Pipe.Collect.toBuffer
+                $ Pipe.FS.read a
+                  >-> Pipe.Node.inEOS (Pipe.Buffer.toString UTF8)
+                  >-/-> Pipe.Node.fromTransform csvDecode
+                  >-/-> Pipe.Node.fromTransform cborEncode
+                  >-> Pipe.Node.unEOS
+              f :: Array Foreign <- liftEffect $ cborDecodeSync cbor
+              ppl <- traverse (liftEither <<< lmap (error <<< show) <<< runExcept <<< readImpl) f
+              ppl `shouldEqual` people'
+
+        around tmpFiles
+          $ it (show n <> " row csv >-> sync csv-parse >-> sync cborEncode") \(a /\ _) -> do
+              liftEffect $ FS.writeTextFile UTF8 a csv
+              cbor :: Buffer <- Pipe.Collect.toBuffer
+                $ Pipe.FS.read a
+                  >-> Pipe.Node.inEOS (Pipe.Buffer.toString UTF8)
+                  >-> sync (Pipe.Node.fromTransform csvDecode)
+                  >-> sync (Pipe.Node.fromTransform cborEncode)
+                  >-> Pipe.Node.unEOS
+              f :: Array Foreign <- liftEffect $ cborDecodeSync cbor
+              ppl <- traverse (liftEither <<< lmap (error <<< show) <<< runExcept <<< readImpl) f
+              ppl `shouldEqual` people'
+
+      around tmpFiles
+        $ it "file >-> sync gzip >-> sync gunzip" \(a /\ _) -> do
+            liftEffect $ FS.writeTextFile UTF8 a $ writeJSON bignums
+            json <- Pipe.Collect.toMonoid
+              $ Pipe.FS.read a
+                >-> sync Pipe.Zlib.gzip
+                >-> sync Pipe.Zlib.gunzip
+                >-> Pipe.Node.unEOS
+                >-> Pipe.Buffer.toString UTF8
+            readJSON json `shouldEqual` (Right bignums)
+
+      around tmpFiles
+        $ it "file >-/-> gzip >-/-> slow >-/-> gunzip" \(a /\ _) -> do
+            liftEffect $ FS.writeTextFile UTF8 a $ writeJSON bignums
+            json <-
+              Pipe.Collect.toMonoid
+              $ Pipe.FS.read a
+                >-/-> Pipe.Zlib.gzip
+                >-/-> Pipe.Node.fromTransform slowTransform
+                >-/-> Pipe.Zlib.gunzip
+                >-> Pipe.Node.unEOS
+                >-> Pipe.Buffer.toString UTF8
+
+            readJSON json `shouldEqual` (Right bignums)
+      around tmpFiles
+        $ it "file >-> sync gzip >-> sync slow >-> sync gunzip" \(a /\ _) -> do
+            liftEffect $ FS.writeTextFile UTF8 a $ writeJSON bignums
+            json <-
+              Pipe.Collect.toMonoid
+              $ Pipe.FS.read a
+                >-> sync Pipe.Zlib.gzip
+                >-> sync (Pipe.Node.fromTransform slowTransform)
+                >-> sync Pipe.Zlib.gunzip
+                >-> Pipe.Node.unEOS
+                >-> Pipe.Buffer.toString UTF8
+
+            readJSON json `shouldEqual` (Right bignums)
       around tmpFile $ it "file >-> discardTransform" \(p :: String) -> do
         liftEffect $ FS.writeTextFile UTF8 p "foo"
         r <- reader p
-        discard' <- liftEffect discardTransform
+        out :: List.List Int <- Pipe.toListM $ r >-/-> Pipe.Node.fromTransform discardTransform >-> Pipe.Node.unEOS
-        out :: List.List Int <- Pipes.toListM $ r >-> S.fromTransform discard' >-> S.unEOS
         out `shouldEqual` List.Nil
       around tmpFile $ it "file >-> charsTransform" \(p :: String) -> do
         liftEffect $ FS.writeTextFile UTF8 p "foo bar"
         r <- reader p
-        chars' <- liftEffect charsTransform
+        out :: List.List String <- Pipe.toListM $ r >-> Pipe.Node.inEOS (Pipe.Buffer.toString UTF8) >-/-> Pipe.Node.fromTransform charsTransform >-> Pipe.Node.unEOS
-        out :: List.List String <- Pipes.toListM $ r >-> S.inEOS (Pipes.Buffer.toString UTF8) >-> S.fromTransform chars' >-> S.unEOS
         out `shouldEqual` List.fromFoldable [ "f", "o", "o", " ", "b", "a", "r" ]