chore: prepare v2.1.4

README.md

@@ -1,81 +1,115 @@
-# purescript-csv-stream
+# purescript-node-stream-pipes
 
-Type-safe bindings for the streaming API of `csv-parse` and `csv-stringify`.
+Interact with node streams in object mode using [`Pipes`]!
 
-## Installing
+## Install
 ```bash
-spago install csv-stream
+spago install node-stream-pipes
-{bun|yarn|npm|pnpm} install csv-parse csv-stringify
 ```
 
-## Examples
+## Usage
-### Stream
+### Node Streams
+#### Raw Streams
+Raw `objectMode` Node streams are represented in `Node.Stream.Object`:
+ - `Writable a` accepts chunks of type `a`
+ - `Readable a` emits chunks of type `a`
+ - `Transform a b` transforms chunks from `a` to `b`
+
+Non-Object streams can also be represented with these types; for example an `fs.WriteStream`
+can be coerced to `Writable Buffer` without issue.
+
+Interop between these types and `Node.Stream` are provided in `Node.Stream.Object`:
+- `unsafeFrom{String,Buffer}{Writable,Readable,Transform}`
+- `unsafeCoerce{Writable,Readable,Transform}`
+
+#### Pipes
+Streams in `Node.Stream.Object` can be converted to `Producer`s, `Consumer`s and `Pipe`s with `Pipes.Node.Stream`:
+ - `fromReadable :: forall a. <Readable a> -> Producer (Maybe a) <Aff> Unit`
+ - `fromWritable :: forall a. <Writable a> -> Consumer (Maybe a) <Aff> Unit`
+ - `fromTransform :: forall a b. <Transform a b> -> Pipe (Maybe a) (Maybe b) <Aff> Unit`
+
+#### EOS Marker
+Normally, pipe computations will not be executed once any computation in a pipeline exits.
+
+To allow for resource cleanup and awareness that the stream is about to close,
+`Maybe a` is used occasionally in this package as an End-of-Stream marker:
+
 ```purescript
-module Main where
+-- foo.txt is "hello, world!\n"
+chunks <- Pipes.Collect.toArray $ Pipes.FS.read "foo.txt" >-> Pipes.Node.Stream.inEOS (Pipes.Buffer.toString UTF8)
+chunks `shouldEqual` [Just "hello, world!\n", Nothing]
+```
+
+Pipes from `a -> b` unaware of EOS can be lifted to `Maybe a -> Maybe b` with `Pipes.Node.Stream.inEOS`.
+
+Producers of `Maybe a` can drop the EOS marker and emit `a` with `Pipes.Node.Stream.unEOS`.
+
+Producers of `a` can have an EOS marker added with `Pipes.Node.Stream.withEOS`.
+
+#### Example
+`Pipes.PassThrough.js`
+```javascript
+import {PassThrough} from 'stream'
+
+export const makePassThrough = () => new PassThrough()
+```
+
+`Pipes.PassThrough.purs`
+```purescript
+module Pipes.PassThrough where
 
 import Prelude
 
 import Effect (Effect)
 import Effect.Class (liftEffect)
-import Effect.Aff (launchAff_)
+import Effect.Aff (Aff)
-import Node.Stream (pipe)
+import Pipes.Core (Pipe)
-import Node.Stream as Stream
+import Node.Stream.Object as ObjectStream
-import Node.Stream.CSV.Stringify as CSV.Stringify
+import Pipes.Node.Stream as Pipes.Node.Stream
-import Node.Stream.CSV.Parse as CSV.Parse
 
-type MyCSVType1 = {a :: Int, b :: Int, bar :: String, baz :: Boolean}
+type PassThroughStream a = ObjectStream.Transform a a
-type MyCSVType2 = {ab :: Int, bar :: String, baz :: Boolean}
 
-atob :: MyCSVType1 -> MyCSVType2
+foreign import makeRaw :: Effect PassThroughStream
-atob {a, b, bar, baz} = {ab: a + b, bar, baz}
 
-myCSV :: String
+passThrough :: forall a. Pipe a a Aff Unit
-myCSV = "a,b,bar,baz\n1,2,\"hello, world!\",true\n3,3,,f"
+passThrough = do
-
+  raw <- liftEffect $ makeRaw
-main :: Effect Unit
+  Pipes.Node.Stream.fromTransform raw
-main = launchAff_ do
-  parser <- liftEffect $ CSV.Parse.make {}
-  stringifier <- liftEffect $ CSV.Stringify.make {}
-
-  input <- liftEffect $ Stream.readableFromString myCSV
-  liftEffect $ Stream.pipe input parser
-
-  records <- CSV.Parse.readAll parser
-  liftEffect $ for_ records \r -> CSV.Stringify.write $ atob r
-  liftEffect $ Stream.end stringifier
-
-  -- "ab,bar,baz\n3,\"hello, world!\",true\n6,,false"
-  csvString <- CSV.Stringify.readAll stringifier
-  pure unit
 ```
 
-### Synchronous
+### Utilities
+This package provides utilities that explicitly use `MonadRec` to ensure stack-safety
+when dealing with producers of large amounts of data.
+
+- `Pipes.Collect` provides stack-safe utilities for executing a pipeline and collecting results into a collection, `Buffer`, `Monoid` etc.
+- `Pipes.Construct` provides stack-safe utilities for creating producers from in-memory collections.
+- `Pipes.Util` provides some miscellaneous utilities missing from `pipes`.
+
+### Zlib
+Pipes for compression & decompression using `zlib` are provided in `Pipes.Node.Zlib`.
+
+### FS
+Read files with:
+- `Pipes.Node.FS.read <path>`
+- `Pipes.Node.FS.read' <WriteStreamOptions> <path>`
+
 ```purescript
-module Main where
+Pipes.Collect.toStringWith UTF8 $ Pipes.Node.FS.read "foo.txt" >-> Pipes.Stream.unEOS
-
-import Prelude
-
-import Effect (Effect)
-import Effect.Class (liftEffect)
-import Effect.Aff (launchAff_)
-import Node.Stream (pipe)
-import Node.Stream as Stream
-import Node.Stream.CSV.Stringify as CSV.Stringify
-import Node.Stream.CSV.Parse as CSV.Parse
-
-type MyCSVType1 = {a :: Int, b :: Int, bar :: String, baz :: Boolean}
-type MyCSVType2 = {ab :: Int, bar :: String, baz :: Boolean}
-
-atob :: MyCSVType1 -> MyCSVType2
-atob {a, b, bar, baz} = {ab: a + b, bar, baz}
-
-myCSV :: String
-myCSV = "a,b,bar,baz\n1,2,\"hello, world!\",true\n3,3,,f"
-
-main :: Effect Unit
-main = launchAff_ do
-  records :: Array MyCSVType1 <- CSV.Parse.parse myCSV
-  -- "ab,bar,baz\n3,\"hello, world!\",true\n6,,false"
-  csvString <- CSV.Stringify.stringify (atob <$> records)
-  pure unit
 ```
+
+Write files with:
+ - `Pipes.Node.FS.write' <WriteStreamOptions> <path>`
+ - `Pipes.Node.FS.trunc <path>`
+ - `Pipes.Node.FS.create <path>`
+ - `Pipes.Node.FS.append <path>`
+
+```purescript
+  Pipes.Stream.withEOS (
+    Pipes.Construct.eachArray ["id,name", "1,henry", "2,suzie"]
+    >-> Pipes.Util.intersperse "\n"
+    >-> Pipes.Buffer.fromString UTF8
+  )
+  >-> Pipes.Node.FS.create "foo.csv"
+```
+
+[`Pipes`]: https://pursuit.purescript.org/packages/purescript-pipes/8.0.0

package.json

@@ -1,11 +1,13 @@
 {
-  "name": "purescript-csv-stream",
+  "name": "purescript-node-stream-pipes",
-  "version": "v1.0.0",
+  "version": "v2.1.4",
+  "type": "module",
   "dependencies": {
-    "csv-parse": "^5.5.5",
+    "csv-parse": "^5.5.6",
-    "csv-stringify": "^6.4.6"
+    "csv-stringify": "^6.5.0"
   },
   "devDependencies": {
+    "cbor-x": "^1.5.9",
     "typescript": "^5.4.5"
   }
 }

spago.lock

@@ -4,19 +4,37 @@ workspace:
       path: ./
       dependencies:
         - aff: ">=7.1.0 <8.0.0"
+        - arrays: ">=7.3.0 <8.0.0"
+        - console: ">=6.1.0 <7.0.0"
         - control: ">=6.0.0 <7.0.0"
+        - datetime: ">=6.1.0 <7.0.0"
         - effect: ">=4.0.0 <5.0.0"
         - either: ">=6.1.0 <7.0.0"
         - exceptions: ">=6.0.0 <7.0.0"
+        - foldable-traversable: ">=6.0.0 <7.0.0"
+        - foreign-object: ">=4.1.0 <5.0.0"
+        - fork: ">=6.0.0 <7.0.0"
+        - lists: ">=7.0.0 <8.0.0"
         - maybe: ">=6.0.0 <7.0.0"
+        - mmorph: ">=7.0.0 <8.0.0"
         - newtype: ">=5.0.0 <6.0.0"
         - node-buffer: ">=9.0.0 <10.0.0"
         - node-event-emitter: ">=3.0.0 <4.0.0"
+        - node-fs: ">=9.1.0 <9.2.0"
+        - node-path: ">=5.0.0 <6.0.0"
         - node-streams: ">=9.0.0 <10.0.0"
+        - node-zlib: ">=0.4.0 <0.5.0"
+        - now: ">=6.0.0 <7.0.0"
+        - ordered-collections: ">=3.2.0 <4.0.0"
+        - parallel: ">=6.0.0 <7.0.0"
         - pipes: ">=8.0.0 <9.0.0"
         - prelude: ">=6.0.1 <7.0.0"
+        - st: ">=6.2.0 <7.0.0"
+        - strings: ">=6.0.1 <7.0.0"
         - tailrec: ">=6.1.0 <7.0.0"
         - transformers: ">=6.0.0 <7.0.0"
+        - tuples: ">=7.0.0 <8.0.0"
+        - unordered-collections: ">=3.1.0 <4.0.0"
         - unsafe-coerce: ">=6.0.0 <7.0.0"
       test_dependencies:
         - console
@@ -26,6 +44,7 @@ workspace:
         - quickcheck
         - simple-json
         - spec
+        - spec-quickcheck
       build_plan:
         - aff
         - ansi
@@ -87,6 +106,7 @@ workspace:
         - safe-coerce
         - simple-json
         - spec
+        - spec-quickcheck
         - st
         - strings
         - tailrec
@@ -95,6 +115,7 @@ workspace:
         - type-equality
         - typelevel-prelude
         - unfoldable
+        - unordered-collections
         - unsafe-coerce
         - variant
   extra_packages: {}
@@ -791,6 +812,16 @@ packages:
       - tailrec
       - transformers
       - tuples
+  spec-quickcheck:
+    type: registry
+    version: 5.0.0
+    integrity: sha256-iE0iThqZCuDGe3pwg5RvqcL8E5cRQ4txDuloCclOsCs=
+    dependencies:
+      - aff
+      - prelude
+      - quickcheck
+      - random
+      - spec
   st:
     type: registry
     version: 6.2.0
@@ -883,6 +914,21 @@ packages:
       - partial
       - prelude
       - tuples
+  unordered-collections:
+    type: registry
+    version: 3.1.0
+    integrity: sha256-H2eQR+ylI+cljz4XzWfEbdF7ee+pnw2IZCeq69AuJ+Q=
+    dependencies:
+      - arrays
+      - enums
+      - functions
+      - integers
+      - lists
+      - prelude
+      - record
+      - tuples
+      - typelevel-prelude
+      - unfoldable
   unsafe-coerce:
     type: registry
     version: 6.0.0

spago.yaml

@@ -1,7 +1,7 @@
 package:
   name: node-stream-pipes
   publish:
-    version: '1.0.0'
+    version: '2.1.4'
     license: 'GPL-3.0-or-later'
     location:
       githubOwner: 'cakekindel'
@@ -11,22 +11,42 @@ package:
     pedanticPackages: true
   dependencies:
     - aff: ">=7.1.0 <8.0.0"
+    - arrays: ">=7.3.0 <8.0.0"
+    - console: ">=6.1.0 <7.0.0"
     - control: ">=6.0.0 <7.0.0"
+    - datetime: ">=6.1.0 <7.0.0"
     - effect: ">=4.0.0 <5.0.0"
     - either: ">=6.1.0 <7.0.0"
     - exceptions: ">=6.0.0 <7.0.0"
+    - foldable-traversable: ">=6.0.0 <7.0.0"
+    - foreign-object: ">=4.1.0 <5.0.0"
+    - fork: ">=6.0.0 <7.0.0"
+    - lists: ">=7.0.0 <8.0.0"
     - maybe: ">=6.0.0 <7.0.0"
+    - mmorph: ">=7.0.0 <8.0.0"
     - newtype: ">=5.0.0 <6.0.0"
     - node-buffer: ">=9.0.0 <10.0.0"
     - node-event-emitter: ">=3.0.0 <4.0.0"
+    - node-fs: ">=9.1.0 <9.2.0"
+    - node-path: ">=5.0.0 <6.0.0"
     - node-streams: ">=9.0.0 <10.0.0"
+    - node-zlib: ">=0.4.0 <0.5.0"
+    - now: ">=6.0.0 <7.0.0"
+    - ordered-collections: ">=3.2.0 <4.0.0"
+    - parallel: ">=6.0.0 <7.0.0"
     - pipes: ">=8.0.0 <9.0.0"
     - prelude: ">=6.0.1 <7.0.0"
+    - st: ">=6.2.0 <7.0.0"
+    - strings: ">=6.0.1 <7.0.0"
     - tailrec: ">=6.1.0 <7.0.0"
     - transformers: ">=6.0.0 <7.0.0"
+    - tuples: ">=7.0.0 <8.0.0"
+    - unordered-collections: ">=3.1.0 <4.0.0"
     - unsafe-coerce: ">=6.0.0 <7.0.0"
   test:
     main: Test.Main
+    build:
+      strict: true
     dependencies:
       - console
       - gen
@@ -35,5 +55,6 @@ package:
       - quickcheck
       - simple-json
       - spec
+      - spec-quickcheck
 workspace:
   extraPackages: {}

src/Node.Stream.Object.js

@@ -1,45 +1,50 @@
 import Stream from "stream";
 
 /** @type {(s: Stream.Readable | Stream.Transform) => () => boolean} */
-export const isReadableImpl = s => () => s.readable
+export const isReadableImpl = (s) => () => s.readable;
+
+/** @type {(s: Stream.Readable | Stream.Transform) => () => number} */
+export const readableLengthImpl = (s) => () => s.readableLength;
 
 /** @type {(s: Stream.Writable | Stream.Readable) => () => boolean} */
-export const isClosedImpl = s => () => s.closed
+export const isClosedImpl = (s) => () => s.closed;
 
 /** @type {(s: Stream.Writable | Stream.Transform) => () => boolean} */
-export const isWritableImpl = s => () => s.writable
+export const isWritableImpl = (s) => () => s.writable;
+
+/** @type {(s: Stream.Writable | Stream.Transform) => () => boolean} */
+export const needsDrainImpl = (s) => () => s.writableNeedDrain;
 
 /** @type {(s: Stream.Readable | Stream.Transform) => () => boolean} */
-export const isReadableEndedImpl = s => () => s.readableEnded
+export const isReadableEndedImpl = (s) => () => s.readableEnded;
 
 /** @type {(s: Stream.Writable | Stream.Transform) => () => boolean} */
-export const isWritableEndedImpl = s => () => s.writableEnded
+export const isWritableEndedImpl = (s) => () => s.writableEnded;
+
+/** @type {(s: Stream.Writable | Stream.Transform) => () => boolean} */
+export const isWritableFinishedImpl = (s) => () => s.writableFinished;
 
 /** @type {(s: Stream.Writable | Stream.Transform) => () => void} */
 export const endImpl = (s) => () => s.end();
 
-/** @type {<WriteResult>(o: {ok: WriteResult, wouldBlock: WriteResult, closed: WriteResult}) => (s: Stream.Writable | Stream.Transform) => (a: unknown) => () => WriteResult} */
+/** @type {<WriteResult>(o: {ok: WriteResult, wouldBlock: WriteResult}) => (s: Stream.Writable | Stream.Transform) => (a: unknown) => () => WriteResult} */
-export const writeImpl = ({ok, wouldBlock, closed}) => (s) => (a) => () => {
+export const writeImpl =
-  if (s.closed || s.writableEnded) {
+  ({ ok, wouldBlock }) =>
-    return closed
+  (s) =>
-  }
+  (a) =>
+  () => {
+    if (s.write(a)) {
+      return ok;
+    } else {
+      return wouldBlock;
+    }
+  };
 
-  if (s.write(a)) {
+/** @type {<ReadResult>(o: {just: (_a: unknown) => ReadResult, wouldBlock: ReadResult}) => (s: Stream.Readable | Stream.Transform) => () => ReadResult} */
-    return ok
-  } else {
-    return wouldBlock
-  }
-}
-
-/** @type {<ReadResult>(o: {just: (_a: unknown) => ReadResult, wouldBlock: ReadResult, closed: ReadResult}) => (s: Stream.Readable | Stream.Transform) => () => ReadResult} */
 export const readImpl =
-  ({ just, closed, wouldBlock }) =>
+  ({ just, wouldBlock }) =>
   (s) =>
   () => {
-    if (s.closed || s.readableEnded) {
-      return closed;
-    }
-
     const a = s.read();
     if (a === null) {
       return wouldBlock;

src/Node.Stream.Object.purs

@@ -2,11 +2,21 @@ module Node.Stream.Object where
 
 import Prelude
 
+import Control.Monad.Error.Class (liftEither)
+import Control.Monad.ST.Class (liftST)
+import Control.Monad.ST.Global (Global)
+import Control.Monad.ST.Ref (STRef)
+import Control.Monad.ST.Ref as STRef
+import Control.Parallel (parOneOf)
 import Data.Either (Either(..))
+import Data.Generic.Rep (class Generic)
+import Data.Maybe (Maybe(..))
+import Data.Show.Generic (genericShow)
 import Effect (Effect)
 import Effect.Aff (Aff, effectCanceler, makeAff)
+import Effect.Aff as Aff
 import Effect.Class (liftEffect)
-import Effect.Exception (Error)
+import Effect.Exception (Error, error)
 import Effect.Uncurried (mkEffectFn1)
 import Node.Buffer (Buffer)
 import Node.EventEmitter (EventHandle(..))
@@ -17,16 +27,29 @@ import Unsafe.Coerce (unsafeCoerce)
 
 data ReadResult a
   = ReadWouldBlock
-  | ReadClosed
   | ReadJust a
 
+derive instance Generic (ReadResult a) _
+derive instance Functor ReadResult
+derive instance Eq a => Eq (ReadResult a)
+instance Show (ReadResult a) where
+  show = genericShow <<< map (const "..")
+
+maybeReadResult :: forall a. ReadResult a -> Maybe a
+maybeReadResult (ReadWouldBlock) = Nothing
+maybeReadResult (ReadJust a) = Just a
+
 data WriteResult
   = WriteWouldBlock
-  | WriteClosed
   | WriteOk
 
-type ReadResultFFI a = { closed :: ReadResult a, wouldBlock :: ReadResult a, just :: a -> ReadResult a }
+derive instance Generic WriteResult _
-type WriteResultFFI = { closed :: WriteResult, wouldBlock :: WriteResult, ok :: WriteResult }
+derive instance Eq WriteResult
+instance Show WriteResult where
+  show = genericShow
+
+type ReadResultFFI a = { wouldBlock :: ReadResult a, just :: a -> ReadResult a }
+type WriteResultFFI = { wouldBlock :: WriteResult, ok :: WriteResult }
 
 foreign import data Writable :: Type -> Type
 foreign import data Readable :: Type -> Type
@@ -39,13 +62,16 @@ foreign import isReadableImpl :: forall s. s -> Effect Boolean
 foreign import isWritableImpl :: forall s. s -> Effect Boolean
 foreign import isReadableEndedImpl :: forall s. s -> Effect Boolean
 foreign import isWritableEndedImpl :: forall s. s -> Effect Boolean
+foreign import isWritableFinishedImpl :: forall s. s -> Effect Boolean
 foreign import isClosedImpl :: forall s. s -> Effect Boolean
+foreign import needsDrainImpl :: forall s. s -> Effect Boolean
+foreign import readableLengthImpl :: forall s. s -> Effect Int
 
 readResultFFI :: forall a. ReadResultFFI a
-readResultFFI = {closed: ReadClosed, wouldBlock: ReadWouldBlock, just: ReadJust}
+readResultFFI = { wouldBlock: ReadWouldBlock, just: ReadJust }
 
 writeResultFFI :: WriteResultFFI
-writeResultFFI = {closed: WriteClosed, wouldBlock: WriteWouldBlock, ok: WriteOk}
+writeResultFFI = { wouldBlock: WriteWouldBlock, ok: WriteOk }
 
 class Stream :: Type -> Constraint
 class Stream s where
@@ -61,25 +87,31 @@ else instance Stream s => Stream s where
   isClosed s = isClosed s
 
 class Stream s <= Read s a | s -> a where
+  readableLength :: s -> Effect Int
   isReadable :: s -> Effect Boolean
   isReadableEnded :: s -> Effect Boolean
   read :: s -> Effect (ReadResult a)
 
 class Stream s <= Write s a | s -> a where
   isWritable :: s -> Effect Boolean
+  needsDrain :: s -> Effect Boolean
   isWritableEnded :: s -> Effect Boolean
+  isWritableFinished :: s -> Effect Boolean
   write :: s -> a -> Effect WriteResult
   end :: s -> Effect Unit
 
 instance Read (Readable a) a where
+  readableLength = readableLengthImpl
   isReadable = isReadableImpl
   isReadableEnded = isReadableEndedImpl
   read = readImpl readResultFFI
 else instance Read (Transform a b) b where
+  readableLength = readableLengthImpl
   isReadable = isReadableImpl
   isReadableEnded = isReadableEndedImpl
   read = readImpl readResultFFI
 else instance (Read s a) => Read s a where
+  readableLength = readableLengthImpl
   isReadable = isReadableImpl
   isReadableEnded = isReadableEndedImpl
   read s = read s
@@ -87,70 +119,109 @@ else instance (Read s a) => Read s a where
 instance Write (Writable a) a where
   isWritable = isWritableImpl
   isWritableEnded = isWritableEndedImpl
+  isWritableFinished = isWritableFinishedImpl
   write s = writeImpl writeResultFFI s
   end = endImpl
+  needsDrain = needsDrainImpl
 else instance Write (Transform a b) a where
   isWritable = isWritableImpl
   isWritableEnded = isWritableEndedImpl
+  isWritableFinished = isWritableFinishedImpl
   write s = writeImpl writeResultFFI s
   end = endImpl
+  needsDrain = needsDrainImpl
 else instance (Write s a) => Write s a where
   isWritable = isWritableImpl
   isWritableEnded = isWritableEndedImpl
+  isWritableFinished = isWritableFinishedImpl
   write s a = write s a
   end s = end s
+  needsDrain = needsDrainImpl
 
-fromBufferReadable :: forall r. Stream.Readable r -> Readable Buffer
+withErrorST :: forall s. Stream s => s -> Effect { cancel :: Effect Unit, error :: STRef Global (Maybe Error) }
-fromBufferReadable = unsafeCoerce
+withErrorST s = do
+  error <- liftST $ STRef.new Nothing
+  cancel <- flip (Event.once errorH) s \e -> void $ liftST $ STRef.write (Just e) error
+  pure { error, cancel }
 
-fromBufferTransform :: Stream.Duplex -> Transform Buffer Buffer
+unsafeCoerceWritable :: forall r a. Stream.Writable r -> Writable a
-fromBufferTransform = unsafeCoerce
+unsafeCoerceWritable = unsafeCoerce
 
-fromBufferWritable :: forall r. Stream.Writable r -> Writable Buffer
+unsafeCoerceReadable :: forall r a. Stream.Readable r -> Readable a
-fromBufferWritable = unsafeCoerce
+unsafeCoerceReadable = unsafeCoerce
 
-fromStringReadable :: forall r. Stream.Readable r -> Readable String
+unsafeCoerceTransform :: forall a b. Stream.Duplex -> Transform a b
-fromStringReadable = unsafeCoerce
+unsafeCoerceTransform = unsafeCoerce
 
-fromStringTransform :: Stream.Duplex -> Transform String String
+unsafeFromBufferReadable :: forall r. Stream.Readable r -> Readable Buffer
-fromStringTransform = unsafeCoerce
+unsafeFromBufferReadable = unsafeCoerce
 
-fromStringWritable :: forall r. Stream.Writable r -> Writable String
+unsafeFromBufferTransform :: forall a. Stream.Duplex -> Transform Buffer a
-fromStringWritable = unsafeCoerce
+unsafeFromBufferTransform = unsafeCoerce
+
+unsafeFromBufferWritable :: forall r. Stream.Writable r -> Writable Buffer
+unsafeFromBufferWritable = unsafeCoerce
+
+unsafeFromStringReadable :: forall r. Stream.Readable r -> Readable String
+unsafeFromStringReadable = unsafeCoerce
+
+unsafeFromStringTransform :: forall a. Stream.Duplex -> Transform String a
+unsafeFromStringTransform = unsafeCoerce
+
+unsafeFromStringWritable :: forall r. Stream.Writable r -> Writable String
+unsafeFromStringWritable = unsafeCoerce
 
 awaitReadableOrClosed :: forall s a. Read s a => s -> Aff Unit
-awaitReadableOrClosed s = do
+awaitReadableOrClosed s = Aff.supervise do
+  fiber <-
+    Aff.forkAff
+    $ parOneOf
+      [ onceAff0 readableH s $> Right unit
+      , onceAff0 closeH s $> Right unit
+      , Left <$> onceAff1 errorH s
+      ]
   closed <- liftEffect $ isClosed s
-  ended <- liftEffect $ isReadableEnded s
+  readEnded <- liftEffect $ isReadableEnded s
   readable <- liftEffect $ isReadable s
-  when (not ended && not closed && not readable) $ makeAff \res -> do
+  length <- liftEffect $ readableLength s
-    cancelClose <- Event.once closeH (res $ Right unit) s
+  if (not closed && not readEnded && readable && length == 0) then
-    cancelError <- Event.once errorH (res <<< Left) s
+    liftEither =<< Aff.joinFiber fiber
-    cancelReadable <- flip (Event.once readableH) s do
+  else
-      cancelClose
+    Aff.killFiber (error "") fiber
-      cancelError
+
-      res $ Right unit
+awaitFinished :: forall s a. Write s a => s -> Aff Unit
-    pure $ effectCanceler do
+awaitFinished s = Aff.supervise do
-      cancelReadable
+  fiber <- Aff.forkAff $ onceAff0 finishH s
-      cancelClose
+  finished <- liftEffect $ isWritableFinished s
-      cancelError
+  if not finished then Aff.joinFiber fiber else Aff.killFiber (error "") fiber
 
 awaitWritableOrClosed :: forall s a. Write s a => s -> Aff Unit
-awaitWritableOrClosed s = do
+awaitWritableOrClosed s = Aff.supervise do
+  fiber <-
+    Aff.forkAff
+    $ parOneOf
+        [ onceAff0 drainH s $> Right unit
+        , onceAff0 closeH s $> Right unit
+        , Left <$> onceAff1 errorH s
+        ]
   closed <- liftEffect $ isClosed s
-  ended <- liftEffect $ isWritableEnded s
+  writeEnded <- liftEffect $ isWritableEnded s
   writable <- liftEffect $ isWritable s
-  when (not closed && not ended && not writable) $ makeAff \res -> do
+  needsDrain <- liftEffect $ needsDrain s
-    cancelClose <- Event.once closeH (res $ Right unit) s
+  if not closed && not writeEnded && writable && needsDrain then
-    cancelError <- Event.once errorH (res <<< Left) s
+    liftEither =<< Aff.joinFiber fiber
-    cancelDrain <- flip (Event.once drainH) s do
+  else
-      cancelClose
+    Aff.killFiber (error "") fiber
-      cancelError
+
-      res $ Right unit
+onceAff0 :: forall e. EventHandle0 e -> e -> Aff Unit
-    pure $ effectCanceler do
+onceAff0 h emitter = makeAff \res -> do
-      cancelDrain
+  cancel <- Event.once h (res $ Right unit) emitter
-      cancelClose
+  pure $ effectCanceler cancel
-      cancelError
+
+onceAff1 :: forall e a. EventHandle1 e a -> e -> Aff a
+onceAff1 h emitter = makeAff \res -> do
+  cancel <- Event.once h (res <<< Right) emitter
+  pure $ effectCanceler cancel
 
 readableH :: forall s a. Read s a => EventHandle0 s
 readableH = EventHandle "readable" identity
@@ -166,3 +237,6 @@ errorH = EventHandle "error" mkEffectFn1
 
 endH :: forall s a. Write s a => EventHandle0 s
 endH = EventHandle "end" identity
+
+finishH :: forall s a. Write s a => EventHandle0 s
+finishH = EventHandle "finish" identity

src/Pipes.Async.purs

@@ -0,0 +1,368 @@
+module Pipes.Async where
+
+import Prelude hiding (join)
+
+import Control.Alternative (class Alternative, empty, guard)
+import Control.Monad.Cont (class MonadTrans)
+import Control.Monad.Error.Class (class MonadError, class MonadThrow, catchError, throwError)
+import Control.Monad.Except (ExceptT, runExceptT)
+import Control.Monad.Fork.Class (class MonadBracket, class MonadFork, fork)
+import Control.Monad.Maybe.Trans (MaybeT(..), runMaybeT)
+import Control.Monad.Morph (class MFunctor, hoist)
+import Control.Monad.Rec.Class (class MonadRec, Step(..), tailRecM)
+import Control.Monad.ST.Class (liftST)
+import Control.Monad.ST.Ref (STRef)
+import Control.Monad.ST.Ref as ST.Ref
+import Control.Monad.Trans.Class (lift)
+import Control.Parallel (class Parallel, parOneOf)
+import Data.Array as Array
+import Data.DateTime.Instant as Instant
+import Data.Either (Either(..), either)
+import Data.Foldable (class Foldable, fold)
+import Data.Generic.Rep (class Generic)
+import Data.Maybe (Maybe(..), fromMaybe, isNothing)
+import Data.Newtype (unwrap)
+import Data.Show.Generic (genericShow)
+import Data.Time.Duration (Milliseconds)
+import Data.Traversable (class Traversable, traverse, traverse_)
+import Data.Tuple.Nested (type (/\), (/\))
+import Effect.Aff.Class (class MonadAff, liftAff)
+import Effect.Class (class MonadEffect, liftEffect)
+import Effect.Console (log)
+import Effect.Now as Now
+import Pipes (await, yield)
+import Pipes.Collect as Collect
+import Pipes.Core (Pipe, Proxy, Producer)
+
+data WriteSignal
+  = WriteSignalOk
+  | WriteSignalEnded
+
+derive instance Generic WriteSignal _
+derive instance Eq WriteSignal
+derive instance Ord WriteSignal
+instance Show WriteSignal where show = genericShow
+
+instance Discard WriteSignal where
+  discard = bind
+
+data ReadSignal
+  = ReadSignalOk
+  | ReadSignalEnded
+
+derive instance Generic ReadSignal _
+derive instance Eq ReadSignal
+derive instance Ord ReadSignal
+instance Show ReadSignal where show = genericShow
+
+instance Discard ReadSignal where
+  discard = bind
+
+data WriteResult
+  = WriteAgain
+  | WriteNeedsDrain
+  | WriteEnded
+
+derive instance Generic WriteResult _
+derive instance Eq WriteResult
+derive instance Ord WriteResult
+instance Show WriteResult where show = genericShow
+
+data ReadResult a
+  = ReadOk a
+  | ReadWouldBlock
+
+derive instance Generic (ReadResult a) _
+derive instance Eq a => Eq (ReadResult a)
+derive instance Ord a => Ord (ReadResult a)
+derive instance Functor ReadResult
+derive instance Foldable ReadResult
+derive instance Traversable ReadResult
+instance Show a => Show (ReadResult a) where show = genericShow
+
+type AsyncIO a b m r =
+  { write :: a -> m WriteResult
+  , read :: m (ReadResult b)
+  , awaitWrite :: m WriteSignal
+  , awaitRead :: m ReadSignal
+  }
+  /\ AsyncPipe a b m r
+
+-- | An `AsyncPipe` is a `Pipe`-like struct that allows
+-- | concurrently reading from a `Producer` and writing to a `Consumer`.
+-- |
+-- | An implementation of `AsyncPipe` for Node `Transform` streams
+-- | is provided in `Pipes.Node.Stream`.
+data AsyncPipe a b m r
+  -- | A pure return value
+  = Pure r
+  -- | An `AsyncPipe` behind a computation
+  | M (m (AsyncPipe a b m r))
+  -- | Interface to write & read from the backing resource
+  | AsyncIO (AsyncIO a b m r)
+
+-- | Modify request / response types
+mapIO :: forall aa ab ba bb m r. Monad m => (ab -> aa) -> (ba -> bb) -> AsyncPipe aa ba m r -> AsyncPipe ab bb m r
+mapIO _ _ (Pure a) = Pure a
+mapIO a b (M m) = M $ mapIO a b <$> m
+mapIO a b (AsyncIO ({write, awaitWrite, read, awaitRead} /\ m)) =
+  AsyncIO $ {write: write <<< a, awaitWrite, read: map b <$> read, awaitRead} /\ mapIO a b m
+
+-- | Modify request / response types
+bindIO :: forall aa ab ba bb m r. Monad m => (ab -> m aa) -> (ba -> m bb) -> AsyncPipe aa ba m r -> AsyncPipe ab bb m r
+bindIO _ _ (Pure a) = Pure a
+bindIO a b (M m) = M $ bindIO a b <$> m
+bindIO a b (AsyncIO ({write, awaitWrite, read, awaitRead} /\ m)) =
+  AsyncIO $ {write: flip bind write <<< a, awaitWrite, read: traverse b =<< read, awaitRead} /\ bindIO a b m
+
+-- | Remove the `AsyncPipe` wrapper by discarding the IO
+stripIO :: forall a b m r. Monad m => AsyncPipe a b m r -> m r
+stripIO (Pure r) = pure r
+stripIO (M m) = m >>= stripIO
+stripIO (AsyncIO (_ /\ m)) = stripIO m
+
+-- | Execute the `AsyncPipe` monad stack until `AsyncIO` is reached (if any)
+getAsyncIO :: forall a b m r. Monad m => AsyncPipe a b m r -> m (Maybe (AsyncIO a b m r))
+getAsyncIO (AsyncIO a) = pure $ Just a
+getAsyncIO (M m) = m >>= getAsyncIO
+getAsyncIO (Pure _) = pure Nothing
+
+instance MonadTrans (AsyncPipe a b) where
+  lift = M <<< map Pure
+
+instance MFunctor (AsyncPipe a b) where
+  hoist _ (Pure a) = Pure a
+  hoist f (M m) = M $ f $ hoist f <$> m
+  hoist f (AsyncIO ({read, write, awaitWrite, awaitRead} /\ m)) =
+    AsyncIO
+      $ { read: f read
+        , write: f <<< write
+        , awaitWrite: f awaitWrite
+        , awaitRead: f awaitRead
+        }
+        /\ hoist f m
+
+instance Monad m => Functor (AsyncPipe a b m) where
+  map f (Pure r) = Pure $ f r
+  map f (M m) = M $ map f <$> m
+  map f (AsyncIO (io /\ m)) = AsyncIO $ io /\ (f <$> m)
+
+instance Monad m => Apply (AsyncPipe a b m) where
+  apply (Pure f) ma = f <$> ma
+  apply (M mf) ma = M $ (_ <*> ma) <$> mf
+  apply (AsyncIO (io /\ mf)) ma = AsyncIO $ io /\ (mf <*> ma)
+
+instance Monad m => Applicative (AsyncPipe a b m) where
+  pure = Pure
+
+instance Monad m => Bind (AsyncPipe a b m) where
+  bind (Pure a) f = f a
+  bind (M ma) f = M $ (_ >>= f) <$> ma
+  bind (AsyncIO (io /\ m)) f = AsyncIO $ io /\ (m >>= f)
+
+instance Monad m => Monad (AsyncPipe a b m)
+
+instance MonadThrow e m => MonadThrow e (AsyncPipe a b m) where
+  throwError = lift <<< throwError
+
+instance MonadError e m => MonadError e (AsyncPipe a b m) where
+  catchError m f = lift $ catchError (stripIO m) (stripIO <<< f)
+
+instance MonadEffect m => MonadEffect (AsyncPipe a b m) where
+  liftEffect = lift <<< liftEffect
+
+instance MonadAff m => MonadAff (AsyncPipe a b m) where
+  liftAff = lift <<< liftAff
+
+-- | Wraps all fields of an `AsyncPipe` with logging to debug
+-- | behavior and timing.
+debug :: forall a b m r. MonadAff m => String -> AsyncPipe (Maybe a) (Maybe b) m r -> AsyncPipe (Maybe a) (Maybe b) m r
+debug c m =
+  let
+    logL :: forall m'. MonadEffect m' => _ -> m' Unit
+    logL msg = liftEffect $ log $ "[" <> c <> "] " <> msg
+    logR :: forall m'. MonadEffect m' => _ -> m' Unit
+    logR msg = liftEffect $ log $ "[" <> c <> "] " <> fold (Array.replicate 20 " ") <> msg
+
+    time :: forall m' a'. MonadEffect m' => m' a' -> m' (Milliseconds /\ a')
+    time ma = do
+      start <- liftEffect Now.now
+      a <- ma
+      end <- liftEffect Now.now
+      pure $ (end `Instant.diff` start) /\ a
+  in
+    flip bind (fromMaybe m)
+    $ runMaybeT do
+      (io /\ done') <- MaybeT $ lift $ getAsyncIO m
+      let
+        write a = do
+          logL "write >"
+          elapsed /\ w <- time $ io.write a
+          logL $ "< write " <> show w <> " (" <> show (unwrap elapsed) <> "ms)"
+          pure w
+
+        read = do
+          logR "read >"
+          elapsed /\ r <- time $ io.read
+          logR $ "< read " <> show (map (const unit) <$> r) <> " (" <> show (unwrap elapsed) <> "ms)"
+          pure r
+
+        awaitWrite = do
+          logL "awaitWrite >"
+          elapsed /\ w <- time $ io.awaitWrite
+          logL $ "< awaitWrite " <> show w <> " (" <> show (unwrap elapsed) <> "ms)"
+          pure w
+
+        awaitRead = do
+          logR "awaitRead >"
+          elapsed /\ r <- time $ io.awaitRead
+          logR $ "< awaitRead " <> show r <> " (" <> show (unwrap elapsed) <> "ms)"
+          pure r
+
+        done = do
+          logL "done >"
+          elapsed /\ r <- time done'
+          logL $ "< done (" <> show (unwrap elapsed) <> "ms)"
+          pure r
+      pure $ AsyncIO $ {write, read, awaitWrite, awaitRead} /\ done
+
+-- | Convert an `AsyncPipe` to a regular `Pipe`.
+-- |
+-- | Rather than two concurrently-running halves (producer & consumer),
+-- | this requires the `AsyncPipe` to occasionally stop `await`ing data
+-- | written by the upstream `Producer` so that it can `yield` to the downstream `Consumer`.
+-- |
+-- | This implementation chooses to prioritize `yield`ing data to the `Consumer` over
+-- | `await`ing written chunks.
+-- |
+-- | Note that using this limits the potential parallelism of the entire pipeline, ex:
+-- |
+-- | ```purs
+-- | Pipe.FS.read "foo.csv"        -- read
+-- |   >-> sync Pipe.CSV.parse     -- parse
+-- |   >-> sync Pipe.CBOR.encode   -- encode
+-- |   >-> Pipe.FS.write "foo.bin" -- write
+-- | ```
+-- |
+-- | In the above example, this is what happens when the pipeline
+-- | is executed:
+-- | 1. `write` asks `encode` "do you have any data yet?" (fast)
+-- | 1. `encode` asks `parse` "do you have any data yet?" (fast)
+-- | 1. `parse` asks `read` "do you have any data yet?" (fast)
+-- | 1. `read` passes 1 chunk to `parse` (fast)
+-- | 1. `parse` blocks until the chunk is parsed (slow)
+-- | 1. `parse` passes 1 chunk to `encode` (fast)
+-- | 1. `encode` blocks until the chunk is encoded (slow)
+-- | 1. `write` writes the block (fast)
+-- |
+-- | For larger workloads, changing this to use `asyncPipe` would be preferable, ex:
+-- | ```purs
+-- | Pipe.FS.read "foo.csv"        -- read
+-- |   >-/-> Pipe.CSV.parse        -- parse
+-- |   >-/-> Pipe.CBOR.encode      -- encode
+-- |   >-> Pipe.FS.write "foo.bin" -- write
+-- | ```
+-- |
+-- | With this change:
+-- | * `read` will pass chunks to `parse` as fast as `parse` allows
+-- | * `parse` will parse chunks and yield them to `encode` as soon as they're ready
+-- | * `encode` will encode chunks and yield them to `write` as soon as they're ready
+sync :: forall a b f p e m r. MonadError e m => Alternative p => Parallel p m => MonadFork f m => MonadAff m => AsyncPipe (Maybe a) (Maybe b) m r -> Pipe (Maybe a) (Maybe b) m r
+sync m =
+  let
+    liftPipe :: forall r'. (Proxy _ _ _ _ m) r' -> ExceptT (Step _ _) (Proxy _ _ _ _ m) r'
+    liftPipe = lift
+
+    liftM :: forall r'. m r' -> ExceptT (Step _ _) (Proxy _ _ _ _ m) r'
+    liftM = liftPipe <<< lift
+  in
+  lift (getAsyncIO m) >>=
+    case _ of
+      Nothing -> lift $ stripIO m
+      Just ({write, awaitWrite, read, awaitRead} /\ done) ->
+         let
+           awaitRW onR onW =
+             liftM (parOneOf [Right <$> awaitWrite, Left <$> awaitRead])
+             >>= either (const onR) onW
+
+           wSignal WriteSignalOk = WriteAgain
+           wSignal WriteSignalEnded = WriteEnded
+
+           tailRecEarly f a = tailRecM (map (either identity identity) <<< runExceptT <<< f) a
+           continue a = throwError (Loop a)
+           break = (Done <$> liftM (stripIO done)) >>= throwError
+         in do
+           flip tailRecEarly WriteAgain \writable -> do
+             rb <- liftM read
+             case rb of
+               ReadWouldBlock
+                 | writable == WriteEnded -> liftM awaitRead *> continue writable
+                 | writable == WriteNeedsDrain -> awaitRW (continue writable) (continue <<< wSignal)
+                 | otherwise -> pure unit
+               ReadOk (Just b) -> liftPipe (yield $ Just b) *> continue writable
+               ReadOk Nothing -> liftPipe (yield Nothing) *> break
+
+             when (writable /= WriteAgain) $ continue writable
+
+             a <- liftPipe await
+             writable' <- liftM $ write a
+             when (isNothing a) $ continue WriteEnded
+             pure $ Loop writable'
+
+-- | Implementation of `(>-/->)`
+-- |
+-- | In the current `MonadFork` "thread", read data from the `AsyncPipe` as it
+-- | is yielded and `yield` to the downstream `Consumer`.
+-- |
+-- | Concurrently, in a separate thread, read data from the upstream `Producer`
+-- | and write to the `AsyncPipe` at max throughput.
+-- |
+-- | If the producing half fails, the error is caught and rethrown.
+-- |
+-- | If the consuming half fails, the error is caught, the producing half is killed, and the error is rethrown.
+pipeAsync
+  :: forall e f m a b
+   . MonadRec m
+  => MonadAff m
+  => MonadBracket e f m
+  => Producer (Maybe a) m Unit
+  -> AsyncPipe (Maybe a) (Maybe b) m Unit
+  -> Producer (Maybe b) m Unit
+pipeAsync prod m =
+  lift (getAsyncIO m)
+    >>= case _ of
+          Nothing -> pure unit
+          Just ({write, read, awaitWrite, awaitRead} /\ done) -> do
+            errST :: STRef _ (Maybe e) <- lift $ liftEffect $ liftST $ ST.Ref.new Nothing
+            killST :: STRef _ Boolean <- lift $ liftEffect $ liftST $ ST.Ref.new false
+
+            let
+              killThread = void $ liftEffect $ liftST $ ST.Ref.write true killST
+              threadKilled = liftEffect $ liftST $ ST.Ref.read killST
+              putThreadError = void <<< liftEffect <<< liftST <<< flip ST.Ref.write errST <<< Just
+              getThreadError = liftEffect $ liftST $ ST.Ref.read errST
+
+              rx a = do
+                killed <- threadKilled
+                guard $ not killed
+                w <- lift $ write a
+                case w of
+                  WriteNeedsDrain -> lift $ void awaitWrite
+                  WriteEnded -> empty
+                  WriteAgain -> pure unit
+
+              spawn = lift <<< fork <<< flip catchError putThreadError
+
+            _thread <- spawn $ void $ runMaybeT $ Collect.foreach rx (hoist lift prod)
+
+            flip tailRecM unit $ const do
+              getThreadError >>= traverse_ throwError
+              rb <- lift read
+              case rb of
+                ReadOk (Just b) -> yield (Just b) $> Loop unit
+                ReadOk Nothing -> killThread *> yield Nothing $> Done unit
+                ReadWouldBlock -> void (lift awaitRead) $> Loop unit
+
+            lift $ stripIO done
+
+infixl 7 pipeAsync as >-/->

src/Pipes.CSV.Parse.purs

@@ -1 +0,0 @@
-module Pipes.CSV.Parse where

src/Pipes.Collect.purs

@@ -0,0 +1,108 @@
+module Pipes.Collect where
+
+import Prelude
+
+import Control.Monad.Rec.Class (class MonadRec, Step(..), tailRecM)
+import Control.Monad.ST.Class (liftST)
+import Data.Array.ST as Array.ST
+import Data.HashMap (HashMap)
+import Data.HashMap as HashMap
+import Data.Hashable (class Hashable)
+import Data.List (List)
+import Data.List as List
+import Data.Map (Map)
+import Data.Map as Map
+import Data.Maybe (Maybe(..), maybe)
+import Data.Tuple.Nested (type (/\), (/\))
+import Effect.Class (class MonadEffect, liftEffect)
+import Foreign.Object (Object)
+import Foreign.Object.ST as Object.ST
+import Foreign.Object.ST.Unsafe as Object.ST.Unsafe
+import Node.Buffer (Buffer)
+import Node.Buffer as Buffer
+import Node.Encoding (Encoding)
+import Pipes.Core (Producer)
+import Pipes.Internal (Proxy(..))
+
+-- | Fold every value produced with a monadic action
+-- |
+-- | Uses `MonadRec`, supporting producers of arbitrary length.
+traverse :: forall a b m. MonadRec m => (b -> a -> m b) -> b -> Producer a m Unit -> m b
+traverse f b0 p0 =
+  flip tailRecM (p0 /\ b0) \(p /\ b) ->
+    case p of
+      Respond a m -> Loop <$> (m unit /\ _) <$> f b a
+      M m -> Loop <$> (_ /\ b) <$> m
+      Request _ _ -> pure $ Done b
+      Pure _ -> pure $ Done b
+
+-- | Fold every value produced
+-- |
+-- | Uses `MonadRec`, supporting producers of arbitrary length.
+fold :: forall a b m. MonadRec m => (b -> a -> b) -> b -> Producer a m Unit -> m b
+fold f b0 p0 = traverse (\b a -> pure $ f b a) b0 p0
+
+-- | Execute a monadic action on every item in a producer.
+-- |
+-- | Uses `MonadRec`, supporting producers of arbitrary length.
+foreach :: forall a m. MonadRec m => (a -> m Unit) -> Producer a m Unit -> m Unit
+foreach f p0 = traverse (\_ a -> f a) unit p0
+
+-- | `append` all emitted values to `mempty`
+toMonoid :: forall a m. Monoid a => MonadRec m => MonadEffect m => Producer a m Unit -> m a
+toMonoid = fold (<>) mempty
+
+-- | Concatenate all buffers to a single buffer, then decode with the
+-- | provided encoding.
+toStringWith :: forall m. MonadRec m => MonadEffect m => Encoding -> Producer Buffer m Unit -> m String
+toStringWith enc = (liftEffect <<< Buffer.toString enc) <=< toBuffer
+
+-- | Concatenate all produced buffers to a single buffer
+toBuffer :: forall m. MonadRec m => MonadEffect m => Producer Buffer m Unit -> m Buffer
+toBuffer p =
+  (liftEffect <<< maybe (Buffer.alloc 0) pure)
+    =<< traverse
+      ( flip \b ->
+          case _ of
+            Just acc -> do
+              new <- liftEffect $ Buffer.concat [ acc, b ]
+              pure $ Just new
+            _ -> pure $ Just b
+      )
+      Nothing
+      p
+
+-- | Collect all values from a `Producer` into an array.
+toArray :: forall a m. MonadRec m => MonadEffect m => Producer a m Unit -> m (Array a)
+toArray p = do
+  st <- liftEffect $ liftST $ Array.ST.new
+  foreach (void <<< liftEffect <<< liftST <<< flip Array.ST.push st) p
+  liftEffect $ liftST $ Array.ST.unsafeFreeze st
+
+-- | Collect all values from a `Producer` into a list.
+-- |
+-- | Reverses the list after collecting, so that values will be
+-- | in the order they were emitted.
+toList :: forall a m. MonadRec m => MonadEffect m => Producer a m Unit -> m (List a)
+toList = map List.reverse <<< fold (flip List.Cons) List.Nil
+
+-- | Collect all values from a `Producer` into a list.
+-- |
+-- | Does not reverse the list after collecting.
+toListRev :: forall a m. MonadRec m => MonadEffect m => Producer a m Unit -> m (List a)
+toListRev = map List.reverse <<< fold (flip List.Cons) List.Nil
+
+-- | Collect all values from a `Producer` into a Javascript Object.
+toObject :: forall a m. MonadRec m => MonadEffect m => Producer (String /\ a) m Unit -> m (Object a)
+toObject p = do
+  st <- liftEffect $ liftST $ Object.ST.new
+  foreach (\(k /\ v) -> void $ liftEffect $ liftST $ Object.ST.poke k v st) p
+  liftEffect $ liftST $ Object.ST.Unsafe.unsafeFreeze st
+
+-- | Collect all values from a `Producer` into a `HashMap`
+toHashMap :: forall k v m. Hashable k => MonadRec m => Producer (k /\ v) m Unit -> m (HashMap k v)
+toHashMap = fold (\map (k /\ v) -> HashMap.insert k v map) HashMap.empty
+
+-- | Collect all values from a `Producer` into a `Map`
+toMap :: forall k v m. Ord k => MonadRec m => Producer (k /\ v) m Unit -> m (Map k v)
+toMap = fold (\map (k /\ v) -> Map.insert k v map) Map.empty

src/Pipes.Construct.purs

@@ -0,0 +1,64 @@
+module Pipes.Construct where
+
+import Prelude
+
+import Control.Monad.Maybe.Trans (MaybeT(..), runMaybeT)
+import Control.Monad.Rec.Class (class MonadRec, Step(..), tailRecM)
+import Control.Monad.ST.Class (liftST)
+import Control.Monad.Trans.Class (lift)
+import Data.Array as Array
+import Data.Array.ST as Array.ST
+import Data.List (List)
+import Data.List as List
+import Data.Map (Map)
+import Data.Map.Internal as Map.Internal
+import Data.Maybe (fromMaybe)
+import Data.Tuple.Nested (type (/\), (/\))
+import Effect.Class (class MonadEffect, liftEffect)
+import Pipes (yield, (>->))
+import Pipes.Core (Producer)
+import Pipes.Prelude as Pipe
+import Pipes.Util as Pipe.Util
+
+-- Producer that will emit monotonically increasing integers
+-- ex `monotonic 0 -> 0 1 2 3 4 5 6 7 ..`
+monotonic :: forall m. MonadRec m => Int -> Producer Int m Unit
+monotonic start = flip tailRecM start \n -> yield n $> Loop (n + 1)
+
+-- Producer that will emit integers from `start` (inclusive) to `end` (exclusive)
+range :: forall m. MonadRec m => Int -> Int -> Producer Int m Unit
+range start end = monotonic start >-> Pipe.take end
+
+-- | Stack-safe producer that yields every value in an Array
+eachArray :: forall a m. MonadRec m => Array a -> Producer a m Unit
+eachArray as = monotonic 0 >-> Pipe.map (Array.index as) >-> Pipe.Util.whileJust
+
+-- | Stack-safe producer that yields every value in a List
+eachList :: forall a m. MonadRec m => List a -> Producer a m Unit
+eachList init =
+  flip tailRecM init \as -> fromMaybe (Done unit) <$> runMaybeT do
+    head <- MaybeT $ pure $ List.head as
+    tail <- MaybeT $ pure $ List.tail as
+    lift $ yield head
+    pure $ Loop tail
+
+-- | Stack-safe producer that yields every value in a Map
+eachMap :: forall k v m. MonadEffect m => MonadRec m => Map k v -> Producer (k /\ v) m Unit
+eachMap init = do
+  stack <- liftEffect $ liftST $ Array.ST.new
+  let
+    push a = void $ liftEffect $ liftST $ Array.ST.push a stack
+    pop = liftEffect $ liftST $ Array.ST.pop stack
+  flip tailRecM init case _ of
+    Map.Internal.Leaf -> fromMaybe (Done unit) <$> runMaybeT do
+      a <- MaybeT pop
+      pure $ Loop a
+    Map.Internal.Node _ _ k v Map.Internal.Leaf Map.Internal.Leaf -> do
+      yield $ k /\ v
+      pure $ Loop Map.Internal.Leaf
+    Map.Internal.Node _ _ k v Map.Internal.Leaf r -> do
+      yield $ k /\ v
+      pure $ Loop r
+    Map.Internal.Node a b k v l r -> do
+      push $ Map.Internal.Node a b k v Map.Internal.Leaf r
+      pure $ Loop l

src/Pipes.Node.Buffer.purs

@@ -0,0 +1,17 @@
+module Pipes.Node.Buffer where
+
+import Prelude
+
+import Control.Monad.Morph (hoist)
+import Effect.Class (class MonadEffect, liftEffect)
+import Node.Buffer (Buffer)
+import Node.Buffer as Buffer
+import Node.Encoding (Encoding)
+import Pipes.Core (Pipe)
+import Pipes.Prelude as Pipes
+
+toString :: forall m. MonadEffect m => Encoding -> Pipe Buffer String m Unit
+toString enc = hoist liftEffect $ Pipes.mapM $ Buffer.toString enc
+
+fromString :: forall m. MonadEffect m => Encoding -> Pipe String Buffer m Unit
+fromString enc = hoist liftEffect $ Pipes.mapM $ flip Buffer.fromString enc

src/Pipes.Node.FS.purs

@@ -0,0 +1,79 @@
+module Pipes.Node.FS where
+
+import Prelude
+
+import Control.Monad.Error.Class (class MonadThrow)
+import Data.Maybe (Maybe)
+import Effect.Aff.Class (class MonadAff)
+import Effect.Class (liftEffect)
+import Effect.Exception (Error)
+import Node.Buffer (Buffer)
+import Node.FS.Stream (WriteStreamOptions, ReadStreamOptions)
+import Node.FS.Stream as FS.Stream
+import Node.Path (FilePath)
+import Node.Stream.Object as O
+import Pipes.Core (Consumer, Producer)
+import Pipes.Node.Stream (fromReadable, fromWritable)
+import Prim.Row (class Union)
+
+-- | Creates a `fs.Writable` stream for the file
+-- | at the given path.
+-- |
+-- | Writing `Nothing` to this pipe will close the stream.
+-- |
+-- | See `Pipes.Node.Stream.withEOS` for converting `Producer a`
+-- | into `Producer (Maybe a)`, emitting `Nothing` before exiting.
+write'
+  :: forall r trash m
+   . Union r trash WriteStreamOptions
+  => MonadAff m
+  => MonadThrow Error m
+  => Record r
+  -> FilePath
+  -> Consumer (Maybe Buffer) m Unit
+write' o p = do
+  w <- liftEffect $ FS.Stream.createWriteStream' p o
+  fromWritable $ O.unsafeCoerceWritable w
+
+-- | Open a file in write mode, failing if the file already exists.
+-- |
+-- | `write' {flags: "wx"}`
+create :: forall m. MonadAff m => MonadThrow Error m => FilePath -> Consumer (Maybe Buffer) m Unit
+create = write' { flags: "wx" }
+
+-- | Open a file in write mode, truncating it if the file already exists.
+-- |
+-- | `write' {flags: "w"}`
+trunc :: forall m. MonadAff m => MonadThrow Error m => FilePath -> Consumer (Maybe Buffer) m Unit
+trunc = write' { flags: "w" }
+
+-- | Open a file in write mode, appending written contents if the file already exists.
+-- |
+-- | `write' {flags: "a"}`
+append :: forall m. MonadAff m => MonadThrow Error m => FilePath -> Consumer (Maybe Buffer) m Unit
+append = write' { flags: "a" }
+
+-- | Creates a `fs.Readable` stream for the file at the given path.
+-- |
+-- | Emits `Nothing` before closing. To opt out of this behavior,
+-- | use `Pipes.Node.Stream.withoutEOS` or `Pipes.Node.Stream.unEOS`.
+read :: forall m. MonadAff m => MonadThrow Error m => FilePath -> Producer (Maybe Buffer) m Unit
+read p = do
+  r <- liftEffect $ FS.Stream.createReadStream p
+  fromReadable $ O.unsafeCoerceReadable r
+
+-- | Creates a `fs.Readable` stream for the file at the given path.
+-- |
+-- | Emits `Nothing` before closing. To opt out of this behavior,
+-- | use `Pipes.Node.Stream.withoutEOS` or `Pipes.Node.Stream.unEOS`.
+read'
+  :: forall r trash m
+   . Union r trash ReadStreamOptions
+  => MonadAff m
+  => MonadThrow Error m
+  => Record r
+  -> FilePath
+  -> Producer (Maybe Buffer) m Unit
+read' opts p = do
+  r <- liftEffect $ FS.Stream.createReadStream' p opts
+  fromReadable $ O.unsafeCoerceReadable r

src/Pipes.Node.Stream.purs

@@ -1,95 +1,180 @@
 module Pipes.Node.Stream where
 
-import Prelude
+import Prelude hiding (join)
 
-import Control.Alternative (empty)
+import Control.Monad.Error.Class (class MonadThrow, throwError)
-import Control.Monad.Maybe.Trans (MaybeT(..), runMaybeT)
+import Control.Monad.Rec.Class (class MonadRec, Step(..), tailRecM)
-import Control.Monad.Rec.Class (whileJust)
+import Control.Monad.ST.Class (liftST)
+import Control.Monad.ST.Ref as ST.Ref
+import Control.Monad.ST.Ref as STRef
 import Control.Monad.Trans.Class (lift)
-import Data.Maybe (Maybe(..))
+import Data.Maybe (Maybe(..), maybe)
-import Data.Newtype (wrap)
+import Data.Traversable (for_, traverse_)
-import Effect.Aff (Aff, delay)
+import Data.Tuple.Nested ((/\))
-import Effect.Aff.Class (liftAff)
+import Effect (Effect)
+import Effect.Aff.Class (class MonadAff, liftAff)
 import Effect.Class (liftEffect)
+import Effect.Exception (Error)
+import Node.Stream.Object (ReadResult(..), WriteResult(..))
 import Node.Stream.Object as O
 import Pipes (await, yield)
-import Pipes.Core (Consumer, Pipe, Producer)
+import Pipes (for) as P
-import Pipes.Internal (Proxy)
+import Pipes.Async (AsyncPipe(..))
-import Pipes.Internal as P.I
+import Pipes.Async as AsyncPipe
+import Pipes.Core (Consumer, Pipe, Producer, Producer_)
+import Pipes.Util (InvokeResult(..), invoke)
 
-type ProxyFFI :: Type -> Type -> Type -> Type -> Type -> Type -> Type
+-- | Convert a `Readable` stream to a `Pipe`.
-type ProxyFFI a' a b' b r pipe =
+-- |
-  { pure :: r -> pipe
+-- | This will yield `Nothing` before exiting, signaling
-  , request :: a' -> (a -> pipe) -> pipe
+-- | End-of-stream.
-  , respond :: b -> (b' -> pipe) -> pipe
+fromReadable :: forall s a m. MonadThrow Error m => MonadAff m => O.Read s a => s -> Producer_ (Maybe a) m Unit
-  }
+fromReadable r =
-
-proxyFFI :: forall m a' a b' b r. ProxyFFI a' a b' b r (Proxy a' a b' b m r)
-proxyFFI = { pure: P.I.Pure, request: P.I.Request, respond: P.I.Respond }
-
-fromReadable :: forall s a. O.Read s a => s -> Producer (Maybe a) Aff Unit
-fromReadable r = whileJust do
-  liftAff $ delay $ wrap 0.0
-  a <- liftEffect $ O.read r
-  case a of
-    O.ReadWouldBlock -> do
-      lift $ O.awaitReadableOrClosed r
-      pure $ Just unit
-    O.ReadClosed -> do
-      yield Nothing
-      pure Nothing
-    O.ReadJust a' -> do
-      yield $ Just a'
-      pure $ Just unit
-
-fromWritable :: forall s a. O.Write s a => s -> Consumer (Maybe a) Aff Unit
-fromWritable w = do
-  whileJust $ runMaybeT do
-    liftAff $ delay $ wrap 0.0
-    a <- MaybeT await
-    res <- liftEffect $ O.write w a
-    case res of
-      O.WriteClosed -> empty
-      O.WriteOk -> pure unit
-      O.WriteWouldBlock -> do
-        liftAff $ O.awaitWritableOrClosed w
-        pure unit
-  liftEffect $ O.end w
-
-fromTransform :: forall a b. O.Transform a b -> Pipe (Maybe a) (Maybe b) Aff Unit
-fromTransform t =
   let
-    read' {exitOnWouldBlock} =
+    cleanup rmErrorListener = do
-      whileJust $ runMaybeT do
+      liftEffect rmErrorListener
-        liftAff $ delay $ wrap 0.0
+      pure $ Done unit
-        res <- liftEffect $ O.read t
-        case res of
-          O.ReadWouldBlock ->
-            if exitOnWouldBlock then do
-              empty
-            else do
-              liftAff $ O.awaitReadableOrClosed t
-              pure unit
-          O.ReadJust b -> do
-            lift $ yield $ Just b
-            pure unit
-          O.ReadClosed -> do
-            lift $ yield Nothing
-            empty
-  in do
-    whileJust $ runMaybeT do
-      liftAff $ delay $ wrap 0.0
 
-      a <- MaybeT await
+    go { error, cancel } = do
-      writeRes <- liftEffect $ O.write t a
+      err <- liftEffect $ liftST $ STRef.read error
+      for_ err throwError
 
-      lift $ read' {exitOnWouldBlock: true}
+      res <- liftEffect $ O.read r
+      case res of
+        O.ReadJust a -> yield (Just a) $> Loop { error, cancel }
+        O.ReadWouldBlock -> do
+          ended <- liftEffect $ O.isReadableEnded r
+          if ended then do
+            yield Nothing
+            cleanup cancel
+          else
+            liftAff (O.awaitReadableOrClosed r) $> Loop { error, cancel }
+  in
+    do
+      e <- liftEffect $ O.withErrorST r
+      tailRecM go e
 
-      case writeRes of
+-- | Convert a `Writable` stream to a `Pipe`.
-        O.WriteOk -> pure unit
+-- |
-        O.WriteClosed -> empty
+-- | When `Nothing` is piped to this, the stream will
-        O.WriteWouldBlock -> do
+-- | be `end`ed, and the pipe will noop if invoked again.
-          liftAff $ O.awaitWritableOrClosed t
+fromWritable :: forall s a m. MonadThrow Error m => MonadAff m => O.Write s a => s -> Consumer (Maybe a) m Unit
-          pure unit
+fromWritable w = do
-    liftEffect $ O.end t
+  { error: errorST, cancel: removeErrorListener } <- liftEffect $ O.withErrorST w
-    read' {exitOnWouldBlock: false}
+
+  let
+    maybeThrow = traverse_ throwError =<< liftEffect (liftST $ STRef.read errorST)
+
+    waitCanWrite = do
+      shouldWait <- liftEffect $ O.needsDrain w
+      when shouldWait $ liftAff $ O.awaitWritableOrClosed w
+
+    cleanup = do
+      liftAff $ O.awaitFinished w
+      maybeThrow
+      liftEffect removeErrorListener
+
+    onEOS = liftEffect (O.end w) *> cleanup $> Done unit
+    onChunk a = liftEffect (O.write w a) $> Loop unit
+
+    go _ = do
+      maybeThrow
+      waitCanWrite
+      ended <- liftEffect $ O.isWritableEnded w
+      if ended then
+        cleanup $> Done unit
+      else
+        await >>= maybe onEOS onChunk
+
+  tailRecM go unit
+
+fromTransformEffect
+  :: forall a b m
+   . MonadThrow Error m
+  => MonadAff m
+  => Effect (O.Transform a b)
+  -> AsyncPipe (Maybe a) (Maybe b) m Unit
+fromTransformEffect = fromTransform <=< liftEffect
+
+-- | Convert a `Transform` stream to an `AsyncPipe`.
+fromTransform
+  :: forall a b m
+   . MonadThrow Error m
+  => MonadAff m
+  => O.Transform a b
+  -> AsyncPipe (Maybe a) (Maybe b) m Unit
+fromTransform stream = do
+    { error: errorST, cancel: removeErrorListener } <- liftEffect $ O.withErrorST stream
+    let
+      rethrow = traverse_ throwError =<< liftEffect (liftST $ ST.Ref.read errorST)
+      cleanup = liftEffect removeErrorListener
+
+      writeSignal =
+        liftEffect (O.isWritableEnded stream)
+        <#> if _ then AsyncPipe.WriteSignalEnded else AsyncPipe.WriteSignalOk
+
+      readSignal =
+        liftEffect (O.isReadableEnded stream)
+        <#> if _ then AsyncPipe.ReadSignalEnded else AsyncPipe.ReadSignalOk
+
+      writeResult WriteOk = AsyncPipe.WriteAgain
+      writeResult WriteWouldBlock = AsyncPipe.WriteNeedsDrain
+
+      readResult (ReadJust a) = AsyncPipe.ReadOk (Just a)
+      readResult ReadWouldBlock = AsyncPipe.ReadWouldBlock
+
+      awaitWritable = liftAff $ O.awaitWritableOrClosed stream
+      awaitReadable = liftAff $ O.awaitReadableOrClosed stream
+
+      awaitWrite = rethrow *> awaitWritable *> writeSignal
+      awaitRead = rethrow *> awaitReadable *> readSignal
+
+      whenReadNotEnded m =
+        liftEffect (O.isReadableEnded stream)
+          >>= if _ then pure $ AsyncPipe.ReadOk Nothing else m
+
+      readNow = readResult <$> liftEffect (O.read stream)
+      writeNow a = writeResult <$> liftEffect (O.write stream a)
+
+      read = rethrow *> whenReadNotEnded readNow
+
+      write Nothing = liftEffect (O.end stream) $> AsyncPipe.WriteEnded
+      write (Just a) = rethrow *> writeNow a
+
+    AsyncIO ({write, awaitWrite, read, awaitRead} /\ cleanup)
+
+-- | Given a `Producer` of values, wrap them in `Just`.
+-- |
+-- | Before the `Producer` exits, emits `Nothing` as an End-of-stream signal.
+withEOS :: forall a m. Monad m => Producer a m Unit -> Producer (Maybe a) m Unit
+withEOS a = do
+  P.for a (yield <<< Just)
+  yield Nothing
+
+-- | Strip a pipeline of the EOS signal
+unEOS :: forall a m. Monad m => Pipe (Maybe a) a m Unit
+unEOS = tailRecM (const $ maybe (pure $ Done unit) (\a -> yield a $> Loop unit) =<< await) unit
+
+-- | Lift a `Pipe a a` to `Pipe (Maybe a) (Maybe a)`.
+-- |
+-- | Allows easily using pipes not concerned with the EOS signal with
+-- | pipes that do need this signal.
+-- |
+-- | (ex. `Pipes.Node.Buffer.toString` doesn't need an EOS signal, but `Pipes.Node.FS.create` does.)
+-- |
+-- | `Just` values will be passed to the pipe, and the response(s) will be wrapped in `Just`.
+-- |
+-- | `Nothing` will bypass the given pipe entirely, and the pipe will not be invoked again.
+inEOS :: forall a b m. MonadRec m => Pipe a b m Unit -> Pipe (Maybe a) (Maybe b) m Unit
+inEOS p = flip tailRecM p \p' -> do
+  ma <- await
+  case ma of
+    Just a -> do
+      res <- lift $ invoke p' a
+      case res of
+        Yielded (as /\ p'') -> do
+          for_ (Just <$> as) yield
+          pure $ Loop p''
+        DidNotYield p'' -> pure $ Loop p''
+        Exited -> yield Nothing $> Done unit
+    _ -> yield Nothing $> Done unit

src/Pipes.Node.Zlib.purs

@@ -0,0 +1,43 @@
+module Pipes.Node.Zlib where
+
+import Prelude
+
+import Control.Monad.Error.Class (class MonadThrow)
+import Data.Maybe (Maybe)
+import Effect (Effect)
+import Effect.Aff.Class (class MonadAff)
+import Effect.Class (liftEffect)
+import Effect.Exception (Error)
+import Node.Buffer (Buffer)
+import Node.Stream.Object as O
+import Node.Zlib as Zlib
+import Node.Zlib.Types (ZlibStream)
+import Pipes.Async (AsyncPipe)
+import Pipes.Node.Stream (fromTransform)
+
+fromZlib :: forall r m. MonadAff m => MonadThrow Error m => Effect (ZlibStream r) -> AsyncPipe (Maybe Buffer) (Maybe Buffer) m Unit
+fromZlib z =
+  do
+    raw <- liftEffect $ Zlib.toDuplex <$> z
+    fromTransform $ O.unsafeCoerceTransform raw
+
+gzip :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe (Maybe Buffer) (Maybe Buffer) m Unit
+gzip = fromZlib Zlib.createGzip
+
+gunzip :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe (Maybe Buffer) (Maybe Buffer) m Unit
+gunzip = fromZlib Zlib.createGunzip
+
+unzip :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe (Maybe Buffer) (Maybe Buffer) m Unit
+unzip = fromZlib Zlib.createUnzip
+
+inflate :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe (Maybe Buffer) (Maybe Buffer) m Unit
+inflate = fromZlib Zlib.createInflate
+
+deflate :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe (Maybe Buffer) (Maybe Buffer) m Unit
+deflate = fromZlib Zlib.createDeflate
+
+brotliCompress :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe (Maybe Buffer) (Maybe Buffer) m Unit
+brotliCompress = fromZlib Zlib.createBrotliCompress
+
+brotliDecompress :: forall m. MonadAff m => MonadThrow Error m => AsyncPipe (Maybe Buffer) (Maybe Buffer) m Unit
+brotliDecompress = fromZlib Zlib.createBrotliDecompress

src/Pipes.String.purs

@@ -0,0 +1,43 @@
+module Pipes.String where
+
+import Prelude
+
+import Control.Monad.Maybe.Trans (MaybeT(..), runMaybeT)
+import Control.Monad.Rec.Class (whileJust)
+import Control.Monad.ST.Class (liftST)
+import Control.Monad.Trans.Class (lift)
+import Data.Array.ST as Array.ST
+import Data.Foldable (fold, traverse_)
+import Data.Maybe (Maybe(..))
+import Data.String (Pattern)
+import Data.String as String
+import Effect.Class (class MonadEffect, liftEffect)
+import Pipes (await, yield)
+import Pipes.Core (Pipe)
+
+-- | Accumulate string chunks until `pat` is seen, then `yield` the buffered
+-- | string up to (and not including) the pattern.
+-- |
+-- | When end-of-stream is reached, yields the remaining buffered string then `Nothing`.
+-- |
+-- | ```
+-- | toList $ yield "foo,bar,baz" >-> split ","
+-- | -- "foo" : "bar" : "baz" : Nil
+-- | ```
+split :: forall m. MonadEffect m => Pattern -> Pipe (Maybe String) (Maybe String) m Unit
+split pat = do
+  buf <- liftEffect $ liftST $ Array.ST.new
+  whileJust $ runMaybeT do
+    chunk <- MaybeT await
+    case String.indexOf pat chunk of
+      Nothing -> void $ liftEffect $ liftST $ Array.ST.push chunk buf
+      Just ix -> do
+        let
+          { before, after } = String.splitAt ix chunk
+        len <- liftEffect $ liftST $ Array.ST.length buf
+        buf' <- liftEffect $ liftST $ Array.ST.splice 0 len [] buf
+        lift $ yield $ Just $ (fold buf') <> before
+        void $ liftEffect $ liftST $ Array.ST.push (String.drop 1 after) buf
+  buf' <- liftEffect $ liftST $ Array.ST.unsafeFreeze buf
+  traverse_ yield (Just <$> String.split pat (fold buf'))
+  yield Nothing

src/Pipes.Util.purs

@@ -0,0 +1,153 @@
+module Pipes.Util where
+
+import Prelude
+
+import Control.Monad.Maybe.Trans (MaybeT(..), runMaybeT)
+import Control.Monad.Rec.Class (class MonadRec, Step(..), forever, tailRecM)
+import Control.Monad.Rec.Class as Rec
+import Control.Monad.ST.Class (liftST)
+import Control.Monad.ST.Ref (STRef)
+import Control.Monad.ST.Ref as STRef
+import Control.Monad.Trans.Class (lift)
+import Data.Array.ST (STArray)
+import Data.Array.ST as Array.ST
+import Data.Either (hush)
+import Data.HashSet as HashSet
+import Data.Hashable (class Hashable, hash)
+import Data.List.NonEmpty (NonEmptyList)
+import Data.Maybe (Maybe(..), fromMaybe)
+import Data.Tuple.Nested (type (/\), (/\))
+import Effect.Class (class MonadEffect, liftEffect)
+import Pipes (await, yield)
+import Pipes as Pipes
+import Pipes.Core (Pipe, Producer)
+import Pipes.Internal (Proxy(..))
+
+-- | Re-yield all `Just`s, and close when `Nothing` is encountered
+whileJust :: forall m a. MonadRec m => Pipe (Maybe a) a m Unit
+whileJust = do
+  first <- await
+  flip tailRecM first $ \ma -> fromMaybe (Done unit) <$> runMaybeT do
+    a <- MaybeT $ pure ma
+    lift $ yield a
+    lift $ Loop <$> await
+
+-- | Yields a separator value `sep` between received values
+-- |
+-- | ```purescript
+-- | toList $ (yield "a" *> yield "b" *> yield "c") >-> intersperse ","
+-- | -- "a" : "," : "b" : "," : "c" : Nil
+-- | ```
+intersperse :: forall m a. MonadEffect m => a -> Pipe (Maybe a) (Maybe a) m Unit
+intersperse sep = do
+  isFirstST <- liftEffect $ liftST $ STRef.new true
+  let
+    getIsFirst = liftEffect $ liftST $ STRef.read isFirstST
+    markNotFirst = void $ liftEffect $ liftST $ STRef.write false isFirstST
+
+  Rec.whileJust $ runMaybeT do
+    a <- MaybeT await
+    isFirst <- getIsFirst
+    if isFirst then markNotFirst else lift $ yield $ Just sep
+    lift $ yield $ Just a
+
+  yield Nothing
+
+-- Pair every emitted value from 2 producers together, exiting when either exits.
+zip :: forall a b m. MonadRec m => Producer a m Unit -> Producer b m Unit -> Producer (a /\ b) m Unit
+zip as bs =
+  flip tailRecM (as /\ bs) \(as' /\ bs') ->
+    fromMaybe (Done unit) <$> runMaybeT do
+      a /\ as'' <- MaybeT $ lift $ hush <$> Pipes.next as'
+      b /\ bs'' <- MaybeT $ lift $ hush <$> Pipes.next bs'
+      lift $ yield $ a /\ b
+      pure $ Loop $ as'' /\ bs''
+
+-- | Accumulate values in chunks of a given size.
+-- |
+-- | If the pipe closes without yielding a multiple of `size` elements,
+-- | the remaining elements are yielded at the end.
+chunked :: forall m a. MonadEffect m => Int -> Pipe (Maybe a) (Maybe (Array a)) m Unit
+chunked size = do
+  chunkST :: STRef _ (STArray _ a) <- liftEffect $ liftST $ STRef.new =<< Array.ST.new
+  let
+    chunkPut a = liftEffect $ liftST do
+      chunkArray <- STRef.read chunkST
+      void $ Array.ST.push a chunkArray
+    chunkLength = liftEffect $ liftST do
+      chunkArray <- STRef.read chunkST
+      Array.ST.length chunkArray
+    chunkTake = liftEffect $ liftST do
+      chunkArray <- STRef.read chunkST
+      void $ flip STRef.write chunkST =<< Array.ST.new
+      Array.ST.unsafeFreeze chunkArray
+
+  Rec.whileJust $ runMaybeT do
+    a <- MaybeT await
+    chunkPut a
+    len <- lift chunkLength
+    when (len >= size) do
+      chunk <- lift chunkTake
+      lift $ yield $ Just chunk
+  len <- chunkLength
+  when (len > 0) do
+    chunk <- chunkTake
+    yield $ Just chunk
+
+  yield Nothing
+
+-- | Equivalent of unix `uniq`, filtering out duplicate values passed to it.
+-- |
+-- | Uses a `HashSet` of hashes of `a`; for `n` elements `awaited`, this pipe
+-- | will occupy O(n) space, and `yield` in O(1) time.
+uniqHash :: forall a m. Hashable a => MonadEffect m => MonadRec m => Pipe a a m Unit
+uniqHash = do
+  seenHashesST <- liftEffect $ liftST $ STRef.new HashSet.empty
+  forever do
+    a <- await
+    seenHashes <- liftEffect $ liftST $ STRef.read seenHashesST
+    when (not $ HashSet.member (hash a) seenHashes) do
+      void $ liftEffect $ liftST $ STRef.modify (HashSet.insert $ hash a) seenHashesST
+      yield a
+
+-- | The result of a single step forward of a pipe.
+data InvokeResult a b m
+  -- | The pipe `await`ed the value, but did not `yield` a response.
+  = DidNotYield (Pipe a b m Unit)
+  -- | The pipe `await`ed the value, and `yield`ed 1 or more responses.
+  | Yielded (NonEmptyList b /\ Pipe a b m Unit)
+  -- | The pipe `await`ed the value, and exited.
+  | Exited
+
+data IntermediateInvokeResult a b m
+  = IDidNotYield (Pipe a b m Unit)
+  | IYielded (NonEmptyList b /\ Pipe a b m Unit)
+  | IDidNotAwait (Pipe a b m Unit)
+
+-- | Pass a single value to a pipe, returning the result of the pipe's invocation.
+invoke :: forall m a b. Monad m => Pipe a b m Unit -> a -> m (InvokeResult a b m)
+invoke m a =
+  let
+    go :: IntermediateInvokeResult a b m -> m (InvokeResult a b m)
+    go (IYielded (as /\ n)) =
+      case n of
+        Request _ _ -> pure $ Yielded $ as /\ n
+        Respond rep f -> go (IYielded $ (as <> pure rep) /\ f unit)
+        M o -> go =<< IYielded <$> (as /\ _) <$> o
+        Pure _ -> pure Exited
+    go (IDidNotYield n) =
+      case n of
+        Request _ _ -> pure $ DidNotYield n
+        Respond rep f -> go (IYielded $ pure rep /\ f unit)
+        M o -> go =<< IDidNotYield <$> o
+        Pure _ -> pure Exited
+    go (IDidNotAwait n) =
+      case n of
+        Request _ f -> go (IDidNotYield (f a))
+        Respond rep f -> go (IYielded $ pure rep /\ f unit)
+        M o -> go =<< IDidNotAwait <$> o
+        Pure _ -> pure Exited
+  in
+    go (IDidNotAwait m)
+
+

test/Test/Common.purs

@@ -0,0 +1,38 @@
+module Test.Common where
+
+import Prelude
+
+import Control.Monad.Error.Class (class MonadError, liftEither, try)
+import Data.Bifunctor (lmap)
+import Data.String.Gen (genAlphaString)
+import Data.Tuple (fst)
+import Data.Tuple.Nested (type (/\), (/\))
+import Effect.Aff (Aff, bracket)
+import Effect.Class (liftEffect)
+import Effect.Exception (Error, error)
+import Node.FS.Sync as FS
+import Pipes.Core (Pipe)
+import Pipes.Prelude as Pipes
+import Simple.JSON (class ReadForeign, class WriteForeign, readJSON, writeJSON)
+import Test.QuickCheck.Gen (randomSampleOne, resize)
+
+tmpFile :: (String -> Aff Unit) -> Aff Unit
+tmpFile f = tmpFiles (f <<< fst)
+
+tmpFiles :: (String /\ String -> Aff Unit) -> Aff Unit
+tmpFiles =
+  let
+    acq = do
+      randa <- liftEffect $ randomSampleOne $ resize 10 genAlphaString
+      randb <- liftEffect $ randomSampleOne $ resize 10 genAlphaString
+      void $ try $ liftEffect $ FS.mkdir ".tmp"
+      pure $ (".tmp/tmp." <> randa) /\ (".tmp/tmp." <> randb)
+    rel (a /\ b) = liftEffect (try (FS.rm a) *> void (try $ FS.rm b))
+  in
+    bracket acq rel
+
+jsonStringify :: forall m a. Monad m => WriteForeign a => Pipe a String m Unit
+jsonStringify = Pipes.map writeJSON
+
+jsonParse :: forall m @a. MonadError Error m => ReadForeign a => Pipe String a m Unit
+jsonParse = Pipes.mapM (liftEither <<< lmap (error <<< show) <<< readJSON)

test/Test/Main.purs

@@ -3,12 +3,21 @@ module Test.Main where
 import Prelude
 
 import Data.Maybe (Maybe(..))
+import Data.Time.Duration (Milliseconds(..))
 import Effect (Effect)
 import Effect.Aff (launchAff_)
+import Test.Pipes.Collect as Test.Pipes.Collect
+import Test.Pipes.Construct as Test.Pipes.Construct
+import Test.Pipes.Node.Buffer as Test.Pipes.Node.Buffer
+import Test.Pipes.Node.FS as Test.Pipes.Node.FS
 import Test.Pipes.Node.Stream as Test.Pipes.Node.Stream
-import Test.Spec.Reporter (consoleReporter, specReporter)
+import Test.Spec.Reporter (specReporter)
 import Test.Spec.Runner (defaultConfig, runSpec')
 
 main :: Effect Unit
-main = launchAff_ $ runSpec' (defaultConfig { timeout = Nothing }) [ specReporter ] do
+main = launchAff_ $ runSpec' (defaultConfig { slow = Milliseconds 0.0, failFast = true, exit = false, timeout = Nothing }) [ specReporter ] do
   Test.Pipes.Node.Stream.spec
+  Test.Pipes.Node.Buffer.spec
+  Test.Pipes.Node.FS.spec
+  Test.Pipes.Collect.spec
+  Test.Pipes.Construct.spec

test/Test/Pipes.Collect.purs

@@ -0,0 +1,111 @@
+module Test.Pipes.Collect where
+
+import Prelude
+
+import Control.Monad.Gen (chooseInt)
+import Control.Monad.Rec.Class (Step(..), tailRecM)
+import Control.Monad.ST as ST
+import Control.Monad.ST.Ref as STRef
+import Data.Array as Array
+import Data.Bifunctor (lmap)
+import Data.HashMap (HashMap)
+import Data.HashMap as HashMap
+import Data.List (List)
+import Data.List as List
+import Data.Map (Map)
+import Data.Map as Map
+import Data.Maybe (Maybe(..))
+import Data.Traversable (traverse)
+import Data.Tuple.Nested (type (/\), (/\))
+import Effect.Aff (Aff)
+import Effect.Class (liftEffect)
+import Effect.Unsafe (unsafePerformEffect)
+import Foreign.Object (Object)
+import Foreign.Object as Object
+import Pipes (yield)
+import Pipes.Collect as Pipes.Collect
+import Pipes.Core (Producer)
+import Test.QuickCheck.Gen (randomSampleOne)
+import Test.Spec (Spec, describe, it)
+import Test.Spec.Assertions (shouldEqual)
+
+testData
+  :: { array :: Array (Int /\ Int)
+     , list :: List (Int /\ Int)
+     , strarray :: Array (String /\ Int)
+     , object :: Object Int
+     , map :: Map Int Int
+     , hashMap :: HashMap Int Int
+     , stream :: Producer (Int /\ Int) Aff Unit
+     , streamStr :: Producer (String /\ Int) Aff Unit
+     }
+testData =
+  unsafePerformEffect $ do
+    array <-
+      flip traverse (Array.range 0 99999) \k -> do
+        v <- liftEffect $ randomSampleOne $ chooseInt 0 99999
+        pure $ k /\ v
+    let
+      strarray = lmap show <$> array
+      object = Object.fromFoldable strarray
+
+      map' :: forall m. m -> (Int -> Int -> m -> m) -> m
+      map' empty insert = ST.run do
+        st <- STRef.new empty
+        ST.foreach array \(k /\ v) -> void $ STRef.modify (insert k v) st
+        STRef.read st
+      hashMap = map' HashMap.empty HashMap.insert
+      map = map' Map.empty Map.insert
+    pure
+      { array
+      , strarray
+      , list: List.fromFoldable array
+      , object
+      , hashMap
+      , map
+      , stream: flip tailRecM 0 \ix -> case Array.index array ix of
+          Just a -> yield a $> Loop (ix + 1)
+          Nothing -> pure $ Done unit
+      , streamStr: flip tailRecM 0 \ix -> case Array.index strarray ix of
+          Just a -> yield a $> Loop (ix + 1)
+          Nothing -> pure $ Done unit
+      }
+
+spec :: Spec Unit
+spec =
+  describe "Test.Pipes.Collect" do
+    describe "toArray" do
+      it "collects an array" do
+        act <- Pipes.Collect.toArray testData.stream
+        act `shouldEqual` testData.array
+      it "empty ok" do
+        act :: Array Int <- Pipes.Collect.toArray (pure unit)
+        act `shouldEqual` []
+    describe "toObject" do
+      it "collects" do
+        act <- Pipes.Collect.toObject $ testData.streamStr
+        act `shouldEqual` testData.object
+      it "empty ok" do
+        act :: Object Int <- Pipes.Collect.toObject (pure unit)
+        act `shouldEqual` Object.empty
+    describe "toMap" do
+      it "collects" do
+        act <- Pipes.Collect.toMap testData.stream
+        act `shouldEqual` testData.map
+      it "empty ok" do
+        act :: Map String Int <- Pipes.Collect.toMap (pure unit)
+        act `shouldEqual` Map.empty
+    describe "toHashMap" do
+      it "collects" do
+        act <- Pipes.Collect.toHashMap testData.stream
+        act `shouldEqual` testData.hashMap
+      it "empty ok" do
+        act :: HashMap String Int <- Pipes.Collect.toHashMap (pure unit)
+        act `shouldEqual` HashMap.empty
+    describe "toList" do
+      it "collects" do
+        act <- Pipes.Collect.toList testData.stream
+        act `shouldEqual` testData.list
+      it "empty ok" do
+        act :: List (String /\ Int) <- Pipes.Collect.toList (pure unit)
+        act `shouldEqual` List.Nil

test/Test/Pipes.Construct.purs

@@ -0,0 +1,58 @@
+module Test.Pipes.Construct where
+
+import Prelude
+
+import Data.Array as Array
+import Data.List as List
+import Data.Map as Map
+import Data.Tuple.Nested (type (/\), (/\))
+import Effect.Class (liftEffect)
+import Pipes.Collect as Pipes.Collect
+import Pipes.Construct as Pipes.Construct
+import Test.Spec (Spec, describe, it)
+import Test.Spec.Assertions (shouldEqual)
+
+spec :: Spec Unit
+spec =
+  describe "Test.Pipes.Construct" do
+    describe "eachMap" do
+      it "empty map" do
+        kvs <- Pipes.Collect.toArray $ Pipes.Construct.eachMap Map.empty
+        kvs `shouldEqual` ([] :: Array (Int /\ Int))
+      it "nonempty map" do
+        let
+          exp = (\n -> n /\ n) <$> Array.range 0 99999
+          map = Map.fromFoldable exp
+        kvs <-
+          liftEffect
+            $ Pipes.Collect.toArray
+            $ Pipes.Construct.eachMap
+            $ map
+        kvs `shouldEqual` exp
+    describe "eachArray" do
+      it "empty array" do
+        kvs <- Pipes.Collect.toArray $ Pipes.Construct.eachArray []
+        kvs `shouldEqual` ([] :: Array Int)
+      it "nonempty array" do
+        let
+          inp = (\n -> n /\ n) <$> Array.range 0 99999
+        kvs <-
+          liftEffect
+            $ Pipes.Collect.toArray
+            $ Pipes.Construct.eachArray
+            $ inp
+        kvs `shouldEqual` inp
+    describe "eachList" do
+      it "empty list" do
+        kvs <- Pipes.Collect.toArray $ Pipes.Construct.eachList List.Nil
+        kvs `shouldEqual` ([] :: Array Int)
+      it "nonempty list" do
+        let
+          inp = (\n -> n /\ n) <$> Array.range 0 99999
+        kvs <-
+          liftEffect
+            $ Pipes.Collect.toArray
+            $ Pipes.Construct.eachList
+            $ List.fromFoldable
+            $ inp
+        kvs `shouldEqual` inp

test/Test/Pipes.Node.Buffer.purs

@@ -0,0 +1,71 @@
+module Test.Pipes.Node.Buffer where
+
+import Prelude
+
+import Control.Monad.Error.Class (catchError)
+import Control.Monad.Gen (chooseInt, sized)
+import Data.Array as Array
+import Data.FoldableWithIndex (forWithIndex_)
+import Data.Int as Int
+import Data.String.Gen (genAsciiString)
+import Data.Tuple (fst, snd)
+import Data.Tuple.Nested ((/\))
+import Effect.Class (liftEffect)
+import Effect.Unsafe (unsafePerformEffect)
+import Node.Buffer (Buffer, BufferValueType(..))
+import Node.Buffer as Buffer
+import Node.Encoding (Encoding(..))
+import Pipes ((>->))
+import Pipes (each) as Pipes
+import Pipes.Core (runEffect) as Pipes
+import Pipes.Node.Buffer as Pipes.Node.Buffer
+import Pipes.Prelude (drain, toListM) as Pipes
+import Test.QuickCheck (class Arbitrary)
+import Test.QuickCheck.Arbitrary (arbitrary)
+import Test.QuickCheck.Gen (randomSample', vectorOf)
+import Test.Spec (Spec, describe, it)
+import Test.Spec.Assertions (fail, shouldEqual)
+
+data BufferJunk = BufferJunk Buffer
+
+instance Arbitrary BufferJunk where
+  arbitrary = sized \s -> do
+    ns <- vectorOf s (chooseInt 0 7)
+    pure $ unsafePerformEffect do
+      buf <- Buffer.alloc s
+      forWithIndex_ ns \ix n -> Buffer.write UInt8 (Int.toNumber n) ix buf
+      pure $ BufferJunk buf
+
+data BufferUTF8 = BufferUTF8 String Buffer
+
+instance Arbitrary BufferUTF8 where
+  arbitrary = do
+    s <- genAsciiString
+    pure $ BufferUTF8 s $ unsafePerformEffect $ Buffer.fromString s UTF8
+
+spec :: Spec Unit
+spec = describe "Pipes.Node.Buffer" do
+  describe "toString" do
+    it "fails when encoding wrong" do
+      vals <- Pipes.each <$> (map \(BufferJunk b) -> b) <$> liftEffect (randomSample' 10 arbitrary)
+      let
+        uut = Pipes.runEffect $ vals >-> Pipes.Node.Buffer.toString UTF8 >-> Pipes.drain
+        ok = do
+          uut
+          fail "Should have thrown"
+        err _ = pure unit
+      catchError ok err
+    it "junk OK in hex" do
+      vals <- Pipes.each <$> (map \(BufferJunk b) -> b) <$> liftEffect (randomSample' 10 arbitrary)
+      Pipes.runEffect $ vals >-> Pipes.Node.Buffer.toString Hex >-> Pipes.drain
+    it "UTF8 ok" do
+      vals <- (map \(BufferUTF8 s b) -> s /\ b) <$> liftEffect (randomSample' 100 arbitrary)
+      let
+        bufs = Pipes.each $ snd <$> vals
+        strs = fst <$> vals
+      act <- Array.fromFoldable <$> Pipes.toListM (bufs >-> Pipes.Node.Buffer.toString UTF8)
+      act `shouldEqual` strs
+  describe "fromString" do
+    it "ok" do
+      vals <- Pipes.each <$> liftEffect (randomSample' 100 genAsciiString)
+      Pipes.runEffect $ vals >-> Pipes.Node.Buffer.fromString UTF8 >-> Pipes.drain

test/Test/Pipes.Node.FS.purs

@@ -0,0 +1,81 @@
+module Test.Pipes.Node.FS where
+
+import Prelude
+
+import Control.Monad.Error.Class (catchError)
+import Data.Foldable (fold, intercalate)
+import Data.Newtype (wrap)
+import Data.Tuple.Nested ((/\))
+import Effect.Class (liftEffect)
+import Node.Encoding (Encoding(..))
+import Node.FS.Sync as FS
+import Pipes (yield, (>->))
+import Pipes.Core (runEffect) as Pipes
+import Pipes.Node.Buffer as Pipes.Node.Buffer
+import Pipes.Node.FS as Pipes.Node.FS
+import Pipes.Node.Stream (inEOS, unEOS, withEOS)
+import Pipes.Prelude (drain, map, toListM) as Pipes
+import Pipes.String as Pipes.String
+import Pipes.Util as Pipes.Util
+import Simple.JSON (writeJSON)
+import Test.Common (jsonParse, tmpFile, tmpFiles)
+import Test.Spec (Spec, around, describe, it)
+import Test.Spec.Assertions (fail, shouldEqual)
+
+spec :: Spec Unit
+spec = describe "Pipes.Node.FS" do
+  describe "read" do
+    around tmpFile $ it "fails if the file does not exist" \p -> do
+      flip catchError (const $ pure unit) do
+        Pipes.runEffect $ Pipes.Node.FS.read p >-> Pipes.drain
+        fail "should have thrown"
+    around tmpFile $ it "reads ok" \p -> do
+      liftEffect $ FS.writeTextFile UTF8 p "foo"
+      s <- fold <$> Pipes.toListM (Pipes.Node.FS.read p >-> unEOS >-> Pipes.Node.Buffer.toString UTF8)
+      s `shouldEqual` "foo"
+  describe "create" do
+    around tmpFile $ it "creates the file when not exists" \p -> do
+      Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.create p
+      contents <- liftEffect $ FS.readTextFile UTF8 p
+      contents `shouldEqual` "foo"
+    around tmpFile $ it "fails if the file already exists" \p -> do
+      liftEffect $ FS.writeTextFile UTF8 p "foo"
+      flip catchError (const $ pure unit) do
+        Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.create p
+        fail "should have thrown"
+  describe "append" do
+    around tmpFile $ it "creates the file when not exists" \p -> do
+      Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.append p
+      contents <- liftEffect $ FS.readTextFile UTF8 p
+      contents `shouldEqual` "foo"
+    around tmpFile $ it "appends" \p -> do
+      Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.append p
+      Pipes.runEffect $ withEOS (yield "\n" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.append p
+      Pipes.runEffect $ withEOS (yield "bar" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.append p
+      contents <- liftEffect $ FS.readTextFile UTF8 p
+      contents `shouldEqual` "foo\nbar"
+  describe "trunc" do
+    around tmpFile $ it "creates the file when not exists" \p -> do
+      Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.trunc p
+      contents <- liftEffect $ FS.readTextFile UTF8 p
+      contents `shouldEqual` "foo"
+    around tmpFile $ it "overwrites contents" \p -> do
+      Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.trunc p
+      Pipes.runEffect $ withEOS (yield "bar" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.trunc p
+      contents <- liftEffect $ FS.readTextFile UTF8 p
+      contents `shouldEqual` "bar"
+  around tmpFiles $ it "json lines >-> parse >-> _.foo >-> write" \(a /\ b) -> do
+    let
+      exp = [ { foo: "a" }, { foo: "bar" }, { foo: "123" } ]
+    liftEffect $ FS.writeTextFile UTF8 a $ intercalate "\n" $ writeJSON <$> exp
+    Pipes.runEffect $
+      Pipes.Node.FS.read a
+        >-> inEOS (Pipes.Node.Buffer.toString UTF8)
+        >-> Pipes.String.split (wrap "\n")
+        >-> inEOS (jsonParse @{ foo :: String })
+        >-> inEOS (Pipes.map _.foo)
+        >-> Pipes.Util.intersperse "\n"
+        >-> inEOS (Pipes.Node.Buffer.fromString UTF8)
+        >-> Pipes.Node.FS.create b
+    act <- liftEffect $ FS.readTextFile UTF8 b
+    act `shouldEqual` "a\nbar\n123"

test/Test/Pipes.Node.Stream.js

@@ -1,4 +1,41 @@
 import Stream from 'stream'
+import * as CBOR from "cbor-x";
+import * as CSVDecode from "csv-parse";
+import * as CSVEncode from "csv-stringify";
+
+export const cborDecode = () => new CBOR.DecoderStream({useRecords: false, allowHalfOpen: true});
+export const cborEncode = () => new CBOR.EncoderStream({useRecords: false, allowHalfOpen: true});
+
+export const cborDecodeSync = a => () => CBOR.decodeMultiple(a);
+export const cborEncodeSync = a => () => CBOR.encode(a, {useRecords: false});
+
+export const csvDecode = () => CSVDecode.parse({columns: true, allowHalfOpen: true})
+export const csvEncode = () => CSVEncode.stringify({header: true, allowHalfOpen: true})
+
+export const discardTransform = () => new Stream.Transform({
+  transform: function(_ck, _enc, cb) {
+    cb()
+  },
+  objectMode: true
+})
+
+export const slowTransform = () => {
+  return new Stream.Transform({
+  transform: function(ck, _enc, cb) {
+    this.push(ck)
+    setTimeout(() => cb(), 4)
+  },
+  objectMode: true
+})
+}
+
+export const charsTransform = () => new Stream.Transform({
+  transform: function(ck, _enc, cb) {
+    ck.split('').filter(s => !!s).forEach(s => this.push(s))
+    cb()
+  },
+  objectMode: true,
+})
 
 /** @type {(a: Array<unknown>) => Stream.Readable}*/
 export const readableFromArray = a => Stream.Readable.from(a)

test/Test/Pipes.Node.Stream.purs

@@ -2,78 +2,72 @@ module Test.Pipes.Node.Stream where
 
 import Prelude
 
-import Control.Monad.Error.Class (liftEither, try)
+import Control.Monad.Cont (lift)
-import Control.Monad.Morph (hoist)
+import Control.Monad.Error.Class (liftEither)
-import Control.Monad.Trans.Class (lift)
+import Control.Monad.Except (runExcept)
 import Data.Array as Array
 import Data.Bifunctor (lmap)
+import Data.Either (Either(..))
 import Data.Foldable (fold, intercalate)
+import Data.FoldableWithIndex (forWithIndex_)
+import Data.FunctorWithIndex (mapWithIndex)
+import Data.Int as Int
 import Data.List ((:))
 import Data.List as List
-import Data.Maybe (Maybe(..), fromMaybe)
+import Data.Maybe (Maybe)
 import Data.Newtype (wrap)
+import Data.Profunctor.Strong (first)
+import Data.String as String
 import Data.String.Gen (genAlphaString)
 import Data.Traversable (for_, traverse)
-import Data.Tuple (fst)
 import Data.Tuple.Nested (type (/\), (/\))
 import Effect (Effect)
-import Effect.Aff (Aff, bracket, delay)
+import Effect.Aff (Aff, delay)
-import Effect.Class (liftEffect)
+import Effect.Class (class MonadEffect, liftEffect)
 import Effect.Exception (error)
+import Effect.Unsafe (unsafePerformEffect)
+import Foreign (Foreign)
 import Node.Buffer (Buffer)
 import Node.Buffer as Buffer
 import Node.Encoding (Encoding(..))
 import Node.FS.Stream as FS.Stream
 import Node.FS.Sync as FS
 import Node.Stream.Object as O
-import Node.Zlib as Zlib
+import Pipes (each) as Pipe
 import Pipes (yield, (>->))
-import Pipes.Core (Consumer, Producer, Pipe, runEffect)
+import Pipes.Async (sync, (>-/->))
-import Pipes.Node.Stream as S
+import Pipes.Collect as Pipe.Collect
-import Pipes.Prelude as Pipe
+import Pipes.Core (Consumer, Producer, runEffect)
-import Simple.JSON (class ReadForeign, class WriteForeign, readJSON, writeJSON)
+import Pipes.Node.Buffer as Pipe.Buffer
+import Pipes.Node.FS as Pipe.FS
+import Pipes.Node.Stream as Pipe.Node
+import Pipes.Node.Zlib as Pipe.Zlib
+import Pipes.Prelude (toListM) as Pipe
+import Simple.JSON (readImpl, readJSON, writeJSON)
+import Test.Common (jsonStringify, tmpFile, tmpFiles)
 import Test.QuickCheck.Arbitrary (arbitrary)
-import Test.QuickCheck.Gen (randomSample', randomSampleOne, resize)
+import Test.QuickCheck.Gen (randomSample')
 import Test.Spec (Spec, around, describe, it)
-import Test.Spec.Assertions (shouldEqual)
+import Test.Spec.Assertions (fail, shouldEqual)
 
 foreign import readableFromArray :: forall @a. Array a -> O.Readable a
+foreign import discardTransform :: forall a b. Effect (O.Transform a b)
+foreign import slowTransform :: forall a b. Effect (O.Transform a b)
+foreign import charsTransform :: Effect (O.Transform String String)
+foreign import cborEncodeSync :: forall a. a -> Effect Buffer
+foreign import cborDecodeSync :: forall a. Buffer -> Effect a
+foreign import cborEncode :: forall a. Effect (O.Transform a Buffer)
+foreign import cborDecode :: forall a. Effect (O.Transform Buffer a)
+foreign import csvEncode :: forall a. Effect (O.Transform a String)
+foreign import csvDecode :: forall a. Effect (O.Transform String a)
 
-str2buf :: Pipe (Maybe String) (Maybe Buffer) Aff Unit
+writer :: forall m. MonadEffect m => String -> m (O.Writable Buffer /\ Consumer (Maybe Buffer) Aff Unit)
-str2buf = hoist liftEffect $ Pipe.mapM (traverse $ flip Buffer.fromString UTF8)
+writer a = do
+  stream <- liftEffect $ O.unsafeCoerceWritable <$> FS.Stream.createWriteStream a
+  pure $ stream /\ Pipe.Node.fromWritable stream
 
-buf2str :: Pipe (Maybe Buffer) (Maybe String) Aff Unit
+reader :: forall m. MonadEffect m => String -> m (Producer (Maybe Buffer) Aff Unit)
-buf2str = hoist liftEffect $ Pipe.mapM (traverse $ Buffer.toString UTF8)
+reader a = liftEffect $ Pipe.Node.fromReadable <$> O.unsafeCoerceReadable <$> FS.Stream.createReadStream a
-
-buf2hex :: Pipe (Maybe Buffer) (Maybe String) Aff Unit
-buf2hex = hoist liftEffect $ Pipe.mapM (traverse $ Buffer.toString Hex)
-
-jsonStringify :: forall a. WriteForeign a => Pipe (Maybe a) (Maybe String) Aff Unit
-jsonStringify = Pipe.map (map writeJSON)
-
-jsonParse :: forall @a. ReadForeign a => Pipe (Maybe String) (Maybe a) Aff Unit
-jsonParse = Pipe.mapM (traverse (liftEither <<< lmap (error <<< show) <<< readJSON))
-
-writer :: String -> Effect (Consumer (Maybe Buffer) Aff Unit)
-writer a = S.fromWritable <$> O.fromBufferWritable <$> FS.Stream.createWriteStream a
-
-reader :: String -> Effect (Producer (Maybe Buffer) Aff Unit)
-reader a = S.fromReadable <$> O.fromBufferReadable <$> FS.Stream.createReadStream a
-
-tmpFile :: (String -> Aff Unit) -> Aff Unit
-tmpFile f = tmpFiles (f <<< fst)
-
-tmpFiles :: (String /\ String -> Aff Unit) -> Aff Unit
-tmpFiles =
-  let
-    acq = do
-      randa <- liftEffect $ randomSampleOne $ resize 10 genAlphaString
-      randb <- liftEffect $ randomSampleOne $ resize 10 genAlphaString
-      void $ try $ liftEffect $ FS.mkdir ".tmp"
-      pure $ ("tmp." <> randa) /\ ("tmp." <> randb)
-    rel (a /\ b) = liftEffect (try (FS.rm a) *> void (try $ FS.rm b))
-  in
-    bracket acq rel
 
 spec :: Spec Unit
 spec =
@@ -81,42 +75,42 @@ spec =
     describe "Readable" do
       describe "Readable.from(<Iterable>)" do
         it "empty" do
-          vals <- List.catMaybes <$> (Pipe.toListM $ S.fromReadable $ readableFromArray @{ foo :: String } [])
+          vals <- Pipe.toListM $ (Pipe.Node.fromReadable $ readableFromArray @{ foo :: String } []) >-> Pipe.Node.unEOS
           vals `shouldEqual` List.Nil
         it "singleton" do
-          vals <- List.catMaybes <$> (Pipe.toListM $ S.fromReadable $ readableFromArray @{ foo :: String } [ { foo: "1" } ])
+          vals <- Pipe.toListM $ (Pipe.Node.fromReadable $ readableFromArray @{ foo :: String } [ { foo: "1" } ]) >-> Pipe.Node.unEOS
           vals `shouldEqual` ({ foo: "1" } : List.Nil)
         it "many elements" do
           let exp = (\n -> { foo: show n }) <$> Array.range 0 100
-          vals <- List.catMaybes <$> (Pipe.toListM $ S.fromReadable $ readableFromArray exp)
+          vals <- Pipe.toListM $ (Pipe.Node.fromReadable $ readableFromArray exp) >-> Pipe.Node.unEOS
           vals `shouldEqual` (List.fromFoldable exp)
     describe "Writable" $ around tmpFile do
       describe "fs.WriteStream" do
         it "pipe to file" \p -> do
-          w <- S.fromWritable <$> O.fromBufferWritable <$> liftEffect (FS.Stream.createWriteStream p)
+          stream <- O.unsafeCoerceWritable <$> liftEffect (FS.Stream.createWriteStream p)
           let
+            w = Pipe.Node.fromWritable stream
             source = do
               buf <- liftEffect $ Buffer.fromString "hello" UTF8
-              yield $ Just buf
+              yield buf
-              yield Nothing
+          runEffect $ Pipe.Node.withEOS source >-> w
-          runEffect $ source >-> w
           contents <- liftEffect $ FS.readTextFile UTF8 p
           contents `shouldEqual` "hello"
+          shouldEqual true =<< liftEffect (O.isWritableEnded stream)
         it "async pipe to file" \p -> do
-          w <- S.fromWritable <$> O.fromBufferWritable <$> liftEffect (FS.Stream.createWriteStream p)
+          w <- Pipe.Node.fromWritable <$> O.unsafeCoerceWritable <$> liftEffect (FS.Stream.createWriteStream p)
           let
             source = do
-              yield $ Just "hello, "
+              yield "hello, "
               lift $ delay $ wrap 5.0
-              yield $ Just "world!"
+              yield "world!"
               lift $ delay $ wrap 5.0
-              yield $ Just " "
+              yield " "
               lift $ delay $ wrap 5.0
-              yield $ Just "this is a "
+              yield "this is a "
               lift $ delay $ wrap 5.0
-              yield $ Just "test."
+              yield "test."
-              yield Nothing
+          runEffect $ Pipe.Node.withEOS (source >-> Pipe.Buffer.fromString UTF8) >-> w
-          runEffect $ source >-> str2buf >-> w
           contents <- liftEffect $ FS.readTextFile UTF8 p
           contents `shouldEqual` "hello, world! this is a test."
         it "chained pipes" \p -> do
@@ -125,34 +119,110 @@ spec =
               str :: String <- genAlphaString
               num :: Int <- arbitrary
               stuff :: Array String <- arbitrary
-              pure {str, num, stuff}
+              pure { str, num, stuff }
-          objs <- liftEffect $ randomSample' 1 obj
+          objs <- liftEffect (randomSample' 1 obj)
           let
             exp = fold (writeJSON <$> objs)
-            objs' = for_ (Just <$> objs) yield *> yield Nothing
+          stream /\ w <- liftEffect $ writer p
-          w <- liftEffect $ writer p
+          runEffect $ Pipe.Node.withEOS (Pipe.each objs >-> jsonStringify >-> Pipe.Buffer.fromString UTF8) >-> w
-          runEffect $ objs' >-> jsonStringify >-> str2buf >-> w
           contents <- liftEffect $ FS.readTextFile UTF8 p
           contents `shouldEqual` exp
+          shouldEqual true =<< liftEffect (O.isWritableEnded stream)
     describe "Transform" do
-      it "gzip" do
+      let
-        let
+        bignums = Array.range 1 1000
-          json = do
+        firstNames = String.split (wrap "\n") $ unsafePerformEffect (FS.readTextFile UTF8 "./test/Test/first_names.txt")
-            yield $ Just $ writeJSON {foo: "bar"}
+        lastNames = String.split (wrap "\n") $ unsafePerformEffect (FS.readTextFile UTF8 "./test/Test/last_names.txt")
-            yield Nothing
+        names n = do
-          exp = "1f8b0800000000000003ab564acbcf57b2524a4a2c52aa0500eff52bfe0d000000"
+          first <- firstNames
-        gzip <- S.fromTransform <$> O.fromBufferTransform <$> liftEffect (Zlib.toDuplex <$> Zlib.createGzip)
+          last <- Array.take (Int.round $ Int.toNumber n / Int.toNumber (Array.length firstNames)) lastNames
-        outs :: List.List String <- List.catMaybes <$> Pipe.toListM (json >-> str2buf >-> gzip >-> buf2hex)
+          pure $ first <> " " <> last
-        fold outs `shouldEqual` exp
+        people n = mapWithIndex (\ix name -> {id: show $ ix + 1, name}) (names n)
-      around tmpFiles
+        peopleCSV n = "id,name\n" <> intercalate "\n" ((\{id, name} -> id <> "," <> name) <$> people n)
-        $ it "file >-> gzip >-> file >-> gunzip" \(a /\ b) -> do
-            liftEffect $ FS.writeTextFile UTF8 a $ writeJSON [1, 2, 3, 4]
-            areader <- liftEffect $ reader a
-            bwriter <- liftEffect $ writer b
-            gzip <- S.fromTransform <$> O.fromBufferTransform <$> liftEffect (Zlib.toDuplex <$> Zlib.createGzip)
-            runEffect $ areader >-> gzip >-> bwriter
 
-            gunzip <- S.fromTransform <$> O.fromBufferTransform <$> liftEffect (Zlib.toDuplex <$> Zlib.createGunzip)
+      for_ [4000, 8000, 32000, 64000, 200000] \n -> do
-            breader <- liftEffect $ reader b
+        let
-            nums <- Pipe.toListM (breader >-> gunzip >-> buf2str >-> jsonParse @(Array Int) >-> Pipe.mapFoldable (fromMaybe []))
+          csv = peopleCSV n
-            Array.fromFoldable nums `shouldEqual` [1, 2, 3, 4]
+          people' = people n
+        around tmpFiles
+          $ it (show n <> " row csv >-/-> csv-parse >-/-> cborEncode") \(a /\ _) -> do
+              liftEffect $ FS.writeTextFile UTF8 a csv
+              cbor :: Buffer <- Pipe.Collect.toBuffer
+                $ Pipe.FS.read a
+                  >-> Pipe.Node.inEOS (Pipe.Buffer.toString UTF8)
+                  >-/-> Pipe.Node.fromTransformEffect csvDecode
+                  >-/-> Pipe.Node.fromTransformEffect cborEncode
+                  >-> Pipe.Node.unEOS
+              f :: Array Foreign <- liftEffect $ cborDecodeSync cbor
+              ppl <- traverse (liftEither <<< lmap (error <<< show) <<< runExcept <<< readImpl) f
+              ppl `shouldEqual` people'
+
+        around tmpFiles
+          $ it (show n <> " row csv >-> sync csv-parse >-> sync cborEncode") \(a /\ _) -> do
+              liftEffect $ FS.writeTextFile UTF8 a csv
+              cbor :: Buffer <- Pipe.Collect.toBuffer
+                $ Pipe.FS.read a
+                  >-> Pipe.Node.inEOS (Pipe.Buffer.toString UTF8)
+                  >-> sync (Pipe.Node.fromTransformEffect csvDecode)
+                  >-> sync (Pipe.Node.fromTransformEffect cborEncode)
+                  >-> Pipe.Node.unEOS
+              f :: Array Foreign <- liftEffect $ cborDecodeSync cbor
+              ppl <- traverse (liftEither <<< lmap (error <<< show) <<< runExcept <<< readImpl) f
+              ppl `shouldEqual` people'
+
+      around tmpFiles
+        $ it "file >-> sync gzip >-> sync gunzip" \(a /\ _) -> do
+            liftEffect $ FS.writeTextFile UTF8 a $ writeJSON bignums
+            json <- Pipe.Collect.toMonoid
+              $ Pipe.FS.read a
+                >-> sync Pipe.Zlib.gzip
+                >-> sync Pipe.Zlib.gunzip
+                >-> Pipe.Node.unEOS
+                >-> Pipe.Buffer.toString UTF8
+            readJSON json `shouldEqual` (Right bignums)
+
+      around tmpFiles
+        $ it "file >-/-> gzip >-/-> slow >-/-> gunzip" \(a /\ _) -> do
+            liftEffect $ FS.writeTextFile UTF8 a $ writeJSON bignums
+            json <-
+              Pipe.Collect.toMonoid
+              $ Pipe.FS.read a
+                >-/-> Pipe.Zlib.gzip
+                >-/-> Pipe.Node.fromTransformEffect slowTransform
+                >-/-> Pipe.Zlib.gunzip
+                >-> Pipe.Node.unEOS
+                >-> Pipe.Buffer.toString UTF8
+
+            readJSON json `shouldEqual` (Right bignums)
+      around tmpFiles
+        $ it "file >-> sync gzip >-> sync slow >-> sync gunzip" \(a /\ _) -> do
+            liftEffect $ FS.writeTextFile UTF8 a $ writeJSON bignums
+            json <-
+              Pipe.Collect.toMonoid
+              $ Pipe.FS.read a
+                >-> sync Pipe.Zlib.gzip
+                >-> sync (Pipe.Node.fromTransformEffect slowTransform)
+                >-> sync Pipe.Zlib.gunzip
+                >-> Pipe.Node.unEOS
+                >-> Pipe.Buffer.toString UTF8
+
+            readJSON json `shouldEqual` (Right bignums)
+      around tmpFile $ it "file >-> discardTransform" \(p :: String) -> do
+        liftEffect $ FS.writeTextFile UTF8 p "foo"
+        r <- reader p
+        out :: List.List Int <-
+          Pipe.toListM
+            $ r
+                >-/-> Pipe.Node.fromTransformEffect discardTransform
+                >-> Pipe.Node.unEOS
+        out `shouldEqual` List.Nil
+      around tmpFile $ it "file >-> charsTransform" \(p :: String) -> do
+        liftEffect $ FS.writeTextFile UTF8 p "foo bar"
+        r <- reader p
+        out :: List.List String <-
+          Pipe.toListM $
+            r
+            >-> Pipe.Node.inEOS (Pipe.Buffer.toString UTF8)
+            >-/-> Pipe.Node.fromTransformEffect charsTransform
+            >-> Pipe.Node.unEOS
+        out `shouldEqual` List.fromFoldable [ "f", "o", "o", " ", "b", "a", "r" ]