chore: prepare v1.4.1

package.json

@@ -1,6 +1,6 @@
 {
   "name": "purescript-node-stream-pipes",
-  "version": "v1.1.0",
+  "version": "v1.4.1",
   "type": "module",
   "dependencies": {
     "csv-parse": "^5.5.5",

spago.lock

@@ -9,8 +9,8 @@ workspace:
         - either: ">=6.1.0 <7.0.0"
         - exceptions: ">=6.0.0 <7.0.0"
         - foldable-traversable: ">=6.0.0 <7.0.0"
-        - foreign-object
-        - lists
+        - foreign-object: ">=4.1.0 <5.0.0"
+        - lists: ">=7.0.0 <8.0.0"
         - maybe: ">=6.0.0 <7.0.0"
         - mmorph: ">=7.0.0 <8.0.0"
         - newtype: ">=5.0.0 <6.0.0"
@@ -20,7 +20,7 @@ workspace:
         - node-path: ">=5.0.0 <6.0.0"
         - node-streams: ">=9.0.0 <10.0.0"
         - node-zlib: ">=0.4.0 <0.5.0"
-        - ordered-collections
+        - ordered-collections: ">=3.2.0 <4.0.0"
         - parallel: ">=6.0.0 <7.0.0"
         - pipes: ">=8.0.0 <9.0.0"
         - prelude: ">=6.0.1 <7.0.0"
@@ -28,8 +28,8 @@ workspace:
         - strings: ">=6.0.1 <7.0.0"
         - tailrec: ">=6.1.0 <7.0.0"
         - transformers: ">=6.0.0 <7.0.0"
-        - tuples
-        - unordered-collections
+        - tuples: ">=7.0.0 <8.0.0"
+        - unordered-collections: ">=3.1.0 <4.0.0"
         - unsafe-coerce: ">=6.0.0 <7.0.0"
       test_dependencies:
         - console

spago.yaml

@@ -1,7 +1,7 @@
 package:
   name: node-stream-pipes
   publish:
-    version: '1.1.0'
+    version: '1.4.1'
     license: 'GPL-3.0-or-later'
     location:
       githubOwner: 'cakekindel'
@@ -10,17 +10,14 @@ package:
     strict: true
     pedanticPackages: true
   dependencies:
-    - foreign-object
-    - lists
-    - ordered-collections
-    - tuples
-    - unordered-collections
     - aff: ">=7.1.0 <8.0.0"
     - arrays: ">=7.3.0 <8.0.0"
     - effect: ">=4.0.0 <5.0.0"
     - either: ">=6.1.0 <7.0.0"
     - exceptions: ">=6.0.0 <7.0.0"
     - foldable-traversable: ">=6.0.0 <7.0.0"
+    - foreign-object: ">=4.1.0 <5.0.0"
+    - lists: ">=7.0.0 <8.0.0"
     - maybe: ">=6.0.0 <7.0.0"
     - mmorph: ">=7.0.0 <8.0.0"
     - newtype: ">=5.0.0 <6.0.0"
@@ -30,6 +27,7 @@ package:
     - node-path: ">=5.0.0 <6.0.0"
     - node-streams: ">=9.0.0 <10.0.0"
     - node-zlib: ">=0.4.0 <0.5.0"
+    - ordered-collections: ">=3.2.0 <4.0.0"
     - parallel: ">=6.0.0 <7.0.0"
     - pipes: ">=8.0.0 <9.0.0"
     - prelude: ">=6.0.1 <7.0.0"
@@ -37,9 +35,13 @@ package:
     - strings: ">=6.0.1 <7.0.0"
     - tailrec: ">=6.1.0 <7.0.0"
     - transformers: ">=6.0.0 <7.0.0"
+    - tuples: ">=7.0.0 <8.0.0"
+    - unordered-collections: ">=3.1.0 <4.0.0"
     - unsafe-coerce: ">=6.0.0 <7.0.0"
   test:
     main: Test.Main
+    build:
+      strict: true
     dependencies:
       - console
       - gen

src/Node.Stream.Object.js

@@ -3,12 +3,18 @@ import Stream from "stream";
 /** @type {(s: Stream.Readable | Stream.Transform) => () => boolean} */
 export const isReadableImpl = (s) => () => s.readable;
 
+/** @type {(s: Stream.Readable | Stream.Transform) => () => number} */
+export const readableLengthImpl = (s) => () => s.readableLength;
+
 /** @type {(s: Stream.Writable | Stream.Readable) => () => boolean} */
 export const isClosedImpl = (s) => () => s.closed;
 
 /** @type {(s: Stream.Writable | Stream.Transform) => () => boolean} */
 export const isWritableImpl = (s) => () => s.writable;
 
+/** @type {(s: Stream.Writable | Stream.Transform) => () => boolean} */
+export const needsDrainImpl = (s) => () => s.writableNeedDrain;
+
 /** @type {(s: Stream.Readable | Stream.Transform) => () => boolean} */
 export const isReadableEndedImpl = (s) => () => s.readableEnded;
 
@@ -18,16 +24,12 @@ export const isWritableEndedImpl = (s) => () => s.writableEnded;
 /** @type {(s: Stream.Writable | Stream.Transform) => () => void} */
 export const endImpl = (s) => () => s.end();
 
-/** @type {<WriteResult>(o: {ok: WriteResult, wouldBlock: WriteResult, closed: WriteResult}) => (s: Stream.Writable | Stream.Transform) => (a: unknown) => () => WriteResult} */
+/** @type {<WriteResult>(o: {ok: WriteResult, wouldBlock: WriteResult}) => (s: Stream.Writable | Stream.Transform) => (a: unknown) => () => WriteResult} */
 export const writeImpl =
-  ({ ok, wouldBlock, closed }) =>
+  ({ ok, wouldBlock }) =>
   (s) =>
   (a) =>
   () => {
-    if (s.closed || s.writableEnded) {
-      return closed;
-    }
-
     if (s.write(a)) {
       return ok;
     } else {
@@ -35,15 +37,11 @@ export const writeImpl =
     }
   };
 
-/** @type {<ReadResult>(o: {just: (_a: unknown) => ReadResult, wouldBlock: ReadResult, closed: ReadResult}) => (s: Stream.Readable | Stream.Transform) => () => ReadResult} */
+/** @type {<ReadResult>(o: {just: (_a: unknown) => ReadResult, wouldBlock: ReadResult}) => (s: Stream.Readable | Stream.Transform) => () => ReadResult} */
 export const readImpl =
-  ({ just, closed, wouldBlock }) =>
+  ({ just, wouldBlock }) =>
   (s) =>
   () => {
-    if (s.closed || s.readableEnded) {
-      return closed;
-    }
-
     const a = s.read();
     if (a === null) {
       return wouldBlock;

src/Node.Stream.Object.purs

@@ -26,7 +26,6 @@ import Unsafe.Coerce (unsafeCoerce)
 
 data ReadResult a
   = ReadWouldBlock
-  | ReadClosed
   | ReadJust a
 
 derive instance Generic (ReadResult a) _
@@ -37,7 +36,6 @@ instance Show (ReadResult a) where
 
 data WriteResult
   = WriteWouldBlock
-  | WriteClosed
   | WriteOk
 
 derive instance Generic WriteResult _
@@ -45,8 +43,8 @@ derive instance Eq WriteResult
 instance Show WriteResult where
   show = genericShow
 
-type ReadResultFFI a = { closed :: ReadResult a, wouldBlock :: ReadResult a, just :: a -> ReadResult a }
-type WriteResultFFI = { closed :: WriteResult, wouldBlock :: WriteResult, ok :: WriteResult }
+type ReadResultFFI a = { wouldBlock :: ReadResult a, just :: a -> ReadResult a }
+type WriteResultFFI = { wouldBlock :: WriteResult, ok :: WriteResult }
 
 foreign import data Writable :: Type -> Type
 foreign import data Readable :: Type -> Type
@@ -60,12 +58,14 @@ foreign import isWritableImpl :: forall s. s -> Effect Boolean
 foreign import isReadableEndedImpl :: forall s. s -> Effect Boolean
 foreign import isWritableEndedImpl :: forall s. s -> Effect Boolean
 foreign import isClosedImpl :: forall s. s -> Effect Boolean
+foreign import needsDrainImpl :: forall s. s -> Effect Boolean
+foreign import readableLengthImpl :: forall s. s -> Effect Int
 
 readResultFFI :: forall a. ReadResultFFI a
-readResultFFI = { closed: ReadClosed, wouldBlock: ReadWouldBlock, just: ReadJust }
+readResultFFI = { wouldBlock: ReadWouldBlock, just: ReadJust }
 
 writeResultFFI :: WriteResultFFI
-writeResultFFI = { closed: WriteClosed, wouldBlock: WriteWouldBlock, ok: WriteOk }
+writeResultFFI = { wouldBlock: WriteWouldBlock, ok: WriteOk }
 
 class Stream :: Type -> Constraint
 class Stream s where
@@ -81,25 +81,30 @@ else instance Stream s => Stream s where
   isClosed s = isClosed s
 
 class Stream s <= Read s a | s -> a where
+  readableLength :: s -> Effect Int
   isReadable :: s -> Effect Boolean
   isReadableEnded :: s -> Effect Boolean
   read :: s -> Effect (ReadResult a)
 
 class Stream s <= Write s a | s -> a where
   isWritable :: s -> Effect Boolean
+  needsDrain :: s -> Effect Boolean
   isWritableEnded :: s -> Effect Boolean
   write :: s -> a -> Effect WriteResult
   end :: s -> Effect Unit
 
 instance Read (Readable a) a where
+  readableLength = readableLengthImpl
   isReadable = isReadableImpl
   isReadableEnded = isReadableEndedImpl
   read = readImpl readResultFFI
 else instance Read (Transform a b) b where
+  readableLength = readableLengthImpl
   isReadable = isReadableImpl
   isReadableEnded = isReadableEndedImpl
   read = readImpl readResultFFI
 else instance (Read s a) => Read s a where
+  readableLength = readableLengthImpl
   isReadable = isReadableImpl
   isReadableEnded = isReadableEndedImpl
   read s = read s
@@ -109,16 +114,19 @@ instance Write (Writable a) a where
   isWritableEnded = isWritableEndedImpl
   write s = writeImpl writeResultFFI s
   end = endImpl
+  needsDrain = needsDrainImpl
 else instance Write (Transform a b) a where
   isWritable = isWritableImpl
   isWritableEnded = isWritableEndedImpl
   write s = writeImpl writeResultFFI s
   end = endImpl
+  needsDrain = needsDrainImpl
 else instance (Write s a) => Write s a where
   isWritable = isWritableImpl
   isWritableEnded = isWritableEndedImpl
   write s a = write s a
   end s = end s
+  needsDrain = needsDrainImpl
 
 withErrorST :: forall s. Stream s => s -> Effect { cancel :: Effect Unit, error :: STRef Global (Maybe Error) }
 withErrorST s = do
@@ -146,18 +154,24 @@ fromStringWritable = unsafeCoerce
 
 awaitReadableOrClosed :: forall s a. Read s a => s -> Aff Unit
 awaitReadableOrClosed s = do
-  closed <- liftEffect $ isClosed s
-  ended <- liftEffect $ isReadableEnded s
   readable <- liftEffect $ isReadable s
-  when (not ended && not closed && not readable)
-    $ liftEither =<< parOneOf [ onceAff0 readableH s $> Right unit, onceAff0 closeH s $> Right unit, Left <$> onceAff1 errorH s ]
+  length <- liftEffect $ readableLength s
+  when (readable && length == 0)
+    $ liftEither
+        =<< parOneOf
+          [ onceAff0 readableH s $> Right unit
+          , onceAff0 closeH s $> Right unit
+          , Left <$> onceAff1 errorH s
+          ]
+
+awaitFinished :: forall s a. Write s a => s -> Aff Unit
+awaitFinished s = onceAff0 finishH s
 
 awaitWritableOrClosed :: forall s a. Write s a => s -> Aff Unit
 awaitWritableOrClosed s = do
-  closed <- liftEffect $ isClosed s
-  ended <- liftEffect $ isWritableEnded s
   writable <- liftEffect $ isWritable s
-  when (not ended && not closed && not writable)
+  needsDrain <- liftEffect $ needsDrain s
+  when (writable && needsDrain)
     $ liftEither =<< parOneOf [ onceAff0 drainH s $> Right unit, onceAff0 closeH s $> Right unit, Left <$> onceAff1 errorH s ]
 
 onceAff0 :: forall e. EventHandle0 e -> e -> Aff Unit
@@ -184,3 +198,6 @@ errorH = EventHandle "error" mkEffectFn1
 
 endH :: forall s a. Write s a => EventHandle0 s
 endH = EventHandle "end" identity
+
+finishH :: forall s a. Write s a => EventHandle0 s
+finishH = EventHandle "finish" identity

src/Pipes.CSV.Parse.purs

@@ -1 +0,0 @@
-module Pipes.CSV.Parse where

src/Pipes.Collect.purs

@@ -2,12 +2,9 @@ module Pipes.Collect where
 
 import Prelude
 
-import Control.Monad.Maybe.Trans (MaybeT(..), runMaybeT)
 import Control.Monad.Rec.Class (class MonadRec, Step(..), tailRecM)
 import Control.Monad.ST.Class (liftST)
-import Control.Monad.Trans.Class (lift)
 import Data.Array.ST as Array.ST
-import Data.Either (hush)
 import Data.HashMap (HashMap)
 import Data.HashMap as HashMap
 import Data.Hashable (class Hashable)
@@ -15,45 +12,59 @@ import Data.List (List)
 import Data.List as List
 import Data.Map (Map)
 import Data.Map as Map
-import Data.Maybe (fromMaybe)
+import Data.Maybe (Maybe(..), maybe)
 import Data.Tuple.Nested (type (/\), (/\))
 import Effect.Class (class MonadEffect, liftEffect)
 import Foreign.Object (Object)
 import Foreign.Object.ST as Object.ST
 import Foreign.Object.ST.Unsafe as Object.ST.Unsafe
-import Pipes (next) as Pipes
+import Node.Buffer (Buffer)
+import Node.Buffer as Buffer
 import Pipes.Core (Producer)
-
--- | Fold every value produced
--- |
--- | Uses `MonadRec`, supporting producers of arbitrary length.
-fold :: forall a b m. MonadRec m => (b -> a -> b) -> b -> Producer a m Unit -> m b
-fold f b p =
-  let
-    insertNext b' p' = runMaybeT do
-      a /\ p'' <- MaybeT $ hush <$> Pipes.next p'
-      pure $ Loop $ f b' a /\ p''
-  in
-    flip tailRecM (b /\ p) \(b' /\ p') -> fromMaybe (Done b') <$> insertNext b' p'
+import Pipes.Internal (Proxy(..))
 
 -- | Fold every value produced with a monadic action
 -- |
 -- | Uses `MonadRec`, supporting producers of arbitrary length.
 traverse :: forall a b m. MonadRec m => (b -> a -> m b) -> b -> Producer a m Unit -> m b
-traverse f b p =
-  let
-    insertNext b' p' = runMaybeT do
-      a /\ p'' <- MaybeT $ hush <$> Pipes.next p'
-      b'' <- lift $ f b' a
-      pure $ Loop $ b'' /\ p''
-  in
-    flip tailRecM (b /\ p) \(b' /\ p') -> fromMaybe (Done b') <$> insertNext b' p'
+traverse f b0 p0 =
+  flip tailRecM (p0 /\ b0) \(p /\ b) ->
+    case p of
+      Respond a m -> do
+        b' <- f b a
+        pure $ Loop $ m unit /\ b'
+      M m -> do
+        n <- m
+        pure $ Loop $ (n /\ b)
+      Request _ _ -> pure $ Done b
+      Pure _ -> pure $ Done b
+
+-- | Fold every value produced
+-- |
+-- | Uses `MonadRec`, supporting producers of arbitrary length.
+fold :: forall a b m. MonadRec m => (b -> a -> b) -> b -> Producer a m Unit -> m b
+fold f b0 p0 = traverse (\b a -> pure $ f b a) b0 p0
 
 -- | Execute a monadic action on every item in a producer.
 -- |
 -- | Uses `MonadRec`, supporting producers of arbitrary length.
 foreach :: forall a m. MonadRec m => (a -> m Unit) -> Producer a m Unit -> m Unit
-foreach f = traverse (const f) unit
+foreach f p0 = traverse (\_ a -> f a) unit p0
+
+-- | Concatenate all produced buffers
+toBuffer :: forall m. MonadRec m => MonadEffect m => Producer Buffer m Unit -> m Buffer
+toBuffer p =
+  (liftEffect <<< maybe (Buffer.alloc 0) pure)
+    =<< traverse
+      ( flip \b ->
+          case _ of
+            Just acc -> do
+              new <- liftEffect $ Buffer.concat [ acc, b ]
+              pure $ Just new
+            _ -> pure $ Just b
+      )
+      Nothing
+      p
 
 -- | Collect all values from a `Producer` into an array.
 toArray :: forall a m. MonadRec m => MonadEffect m => Producer a m Unit -> m (Array a)

src/Pipes.Construct.purs

@@ -0,0 +1,64 @@
+module Pipes.Construct where
+
+import Prelude
+
+import Control.Monad.Maybe.Trans (MaybeT(..), runMaybeT)
+import Control.Monad.Rec.Class (class MonadRec, Step(..), tailRecM)
+import Control.Monad.ST.Class (liftST)
+import Control.Monad.Trans.Class (lift)
+import Data.Array as Array
+import Data.Array.ST as Array.ST
+import Data.List (List)
+import Data.List as List
+import Data.Map (Map)
+import Data.Map.Internal as Map.Internal
+import Data.Maybe (fromMaybe)
+import Data.Tuple.Nested (type (/\), (/\))
+import Effect.Class (class MonadEffect, liftEffect)
+import Pipes (yield, (>->))
+import Pipes.Core (Producer)
+import Pipes.Prelude as Pipe
+import Pipes.Util as Pipe.Util
+
+-- Producer that will emit monotonically increasing integers
+-- ex `monotonic 0 -> 0 1 2 3 4 5 6 7 ..`
+monotonic :: forall m. MonadRec m => Int -> Producer Int m Unit
+monotonic start = flip tailRecM start \n -> yield n $> Loop (n + 1)
+
+-- Producer that will emit integers from `start` (inclusive) to `end` (exclusive)
+range :: forall m. MonadRec m => Int -> Int -> Producer Int m Unit
+range start end = monotonic start >-> Pipe.take end
+
+-- | Stack-safe producer that yields every value in an Array
+eachArray :: forall a m. MonadRec m => Array a -> Producer a m Unit
+eachArray as = monotonic 0 >-> Pipe.map (Array.index as) >-> Pipe.Util.whileJust
+
+-- | Stack-safe producer that yields every value in a List
+eachList :: forall a m. MonadRec m => List a -> Producer a m Unit
+eachList init =
+  flip tailRecM init \as -> fromMaybe (Done unit) <$> runMaybeT do
+    head <- MaybeT $ pure $ List.head as
+    tail <- MaybeT $ pure $ List.tail as
+    lift $ yield head
+    pure $ Loop tail
+
+-- | Stack-safe producer that yields every value in a Map
+eachMap :: forall k v m. MonadEffect m => MonadRec m => Map k v -> Producer (k /\ v) m Unit
+eachMap init = do
+  stack <- liftEffect $ liftST $ Array.ST.new
+  let
+    push a = void $ liftEffect $ liftST $ Array.ST.push a stack
+    pop = liftEffect $ liftST $ Array.ST.pop stack
+  flip tailRecM init case _ of
+    Map.Internal.Leaf -> fromMaybe (Done unit) <$> runMaybeT do
+      a <- MaybeT pop
+      pure $ Loop a
+    Map.Internal.Node _ _ k v Map.Internal.Leaf Map.Internal.Leaf -> do
+      yield $ k /\ v
+      pure $ Loop Map.Internal.Leaf
+    Map.Internal.Node _ _ k v Map.Internal.Leaf r -> do
+      yield $ k /\ v
+      pure $ Loop r
+    Map.Internal.Node a b k v l r -> do
+      push $ Map.Internal.Node a b k v Map.Internal.Leaf r
+      pure $ Loop l

src/Pipes.Node.FS.purs

@@ -2,11 +2,13 @@ module Pipes.Node.FS where
 
 import Prelude
 
+import Control.Monad.Error.Class (class MonadThrow)
 import Data.Maybe (Maybe)
-import Effect.Aff (Aff)
+import Effect.Aff.Class (class MonadAff)
 import Effect.Class (liftEffect)
+import Effect.Exception (Error)
 import Node.Buffer (Buffer)
-import Node.FS.Stream (WriteStreamOptions)
+import Node.FS.Stream (WriteStreamOptions, ReadStreamOptions)
 import Node.FS.Stream as FS.Stream
 import Node.Path (FilePath)
 import Node.Stream.Object as O
@@ -22,11 +24,13 @@ import Prim.Row (class Union)
 -- | See `Pipes.Node.Stream.withEOS` for converting `Producer a`
 -- | into `Producer (Maybe a)`, emitting `Nothing` before exiting.
 write
-  :: forall r trash
+  :: forall r trash m
    . Union r trash WriteStreamOptions
+  => MonadAff m
+  => MonadThrow Error m
   => Record r
   -> FilePath
-  -> Consumer (Maybe Buffer) Aff Unit
+  -> Consumer (Maybe Buffer) m Unit
 write o p = do
   w <- liftEffect $ FS.Stream.createWriteStream' p o
   fromWritable $ O.fromBufferWritable w
@@ -34,26 +38,42 @@ write o p = do
 -- | Open a file in write mode, failing if the file already exists.
 -- |
 -- | `write {flags: "wx"}`
-create :: FilePath -> Consumer (Maybe Buffer) Aff Unit
+create :: forall m. MonadAff m => MonadThrow Error m => FilePath -> Consumer (Maybe Buffer) m Unit
 create = write { flags: "wx" }
 
 -- | Open a file in write mode, truncating it if the file already exists.
 -- |
 -- | `write {flags: "w"}`
-truncate :: FilePath -> Consumer (Maybe Buffer) Aff Unit
+truncate :: forall m. MonadAff m => MonadThrow Error m => FilePath -> Consumer (Maybe Buffer) m Unit
 truncate = write { flags: "w" }
 
 -- | Open a file in write mode, appending written contents if the file already exists.
 -- |
 -- | `write {flags: "a"}`
-append :: FilePath -> Consumer (Maybe Buffer) Aff Unit
+append :: forall m. MonadAff m => MonadThrow Error m => FilePath -> Consumer (Maybe Buffer) m Unit
 append = write { flags: "a" }
 
 -- | Creates a `fs.Readable` stream for the file at the given path.
 -- |
 -- | Emits `Nothing` before closing. To opt out of this behavior,
 -- | use `Pipes.Node.Stream.withoutEOS` or `Pipes.Node.Stream.unEOS`.
-read :: FilePath -> Producer (Maybe Buffer) Aff Unit
+read :: forall m. MonadAff m => MonadThrow Error m => FilePath -> Producer (Maybe Buffer) m Unit
 read p = do
   r <- liftEffect $ FS.Stream.createReadStream p
   fromReadable $ O.fromBufferReadable r
+
+-- | Creates a `fs.Readable` stream for the file at the given path.
+-- |
+-- | Emits `Nothing` before closing. To opt out of this behavior,
+-- | use `Pipes.Node.Stream.withoutEOS` or `Pipes.Node.Stream.unEOS`.
+read'
+  :: forall r trash m
+   . Union r trash ReadStreamOptions
+  => MonadAff m
+  => MonadThrow Error m
+  => Record r
+  -> FilePath
+  -> Producer (Maybe Buffer) m Unit
+read' opts p = do
+  r <- liftEffect $ FS.Stream.createReadStream' p opts
+  fromReadable $ O.fromBufferReadable r

src/Pipes.Node.Stream.purs

@@ -1,29 +1,32 @@
 module Pipes.Node.Stream where
 
-import Prelude
+import Prelude hiding (join)
 
-import Control.Monad.Error.Class (throwError)
-import Control.Monad.Rec.Class (Step(..), tailRecM, whileJust)
+import Control.Monad.Error.Class (class MonadThrow, throwError)
+import Control.Monad.Rec.Class (class MonadRec, Step(..), tailRecM)
 import Control.Monad.ST.Class (liftST)
 import Control.Monad.ST.Ref as STRef
 import Control.Monad.Trans.Class (lift)
-import Data.Maybe (Maybe(..), maybe)
+import Data.Maybe (Maybe(..))
 import Data.Newtype (wrap)
 import Data.Traversable (for_)
-import Effect.Aff (Aff, delay)
-import Effect.Aff.Class (liftAff)
+import Data.Tuple.Nested ((/\))
+import Effect.Aff (delay)
+import Effect.Aff.Class (class MonadAff, liftAff)
 import Effect.Class (liftEffect)
+import Effect.Exception (Error)
 import Node.Stream.Object as O
-import Pipes (await, yield, (>->))
+import Pipes (await, yield)
 import Pipes (for) as P
 import Pipes.Core (Consumer, Pipe, Producer, Producer_)
-import Pipes.Prelude (mapFoldable, map) as P
+import Pipes.Prelude (mapFoldable) as P
+import Pipes.Util (InvokeResult(..), invoke)
 
 -- | Convert a `Readable` stream to a `Pipe`.
 -- |
 -- | This will yield `Nothing` before exiting, signaling
 -- | End-of-stream.
-fromReadable :: forall s a. O.Read s a => s -> Producer_ (Maybe a) Aff Unit
+fromReadable :: forall s a m. MonadThrow Error m => MonadAff m => O.Read s a => s -> Producer_ (Maybe a) m Unit
 fromReadable r =
   let
     cleanup rmErrorListener = do
@@ -31,15 +34,19 @@ fromReadable r =
       pure $ Done unit
 
     go { error, cancel } = do
-      liftAff $ delay $ wrap 0.0
       err <- liftEffect $ liftST $ STRef.read error
       for_ err throwError
 
       res <- liftEffect $ O.read r
       case res of
         O.ReadJust a -> yield (Just a) $> Loop { error, cancel }
-        O.ReadWouldBlock -> lift (O.awaitReadableOrClosed r) $> Loop { error, cancel }
-        O.ReadClosed -> yield Nothing *> cleanup cancel
+        O.ReadWouldBlock -> do
+          ended <- liftEffect $ O.isReadableEnded r
+          if ended then do
+            yield Nothing
+            cleanup cancel
+          else
+            liftAff (O.awaitReadableOrClosed r) $> Loop { error, cancel }
   in
     do
       e <- liftEffect $ O.withErrorST r
@@ -49,30 +56,31 @@ fromReadable r =
 -- |
 -- | When `Nothing` is piped to this, the stream will
 -- | be `end`ed, and the pipe will noop if invoked again.
-fromWritable :: forall s a. O.Write s a => s -> Consumer (Maybe a) Aff Unit
+fromWritable :: forall s a m. MonadThrow Error m => MonadAff m => O.Write s a => s -> Consumer (Maybe a) m Unit
 fromWritable w =
   let
     cleanup rmErrorListener = do
       liftEffect rmErrorListener
       liftEffect $ O.end w
+      liftAff $ O.awaitFinished w
       pure $ Done unit
 
     go { error, cancel } = do
-      liftAff $ delay $ wrap 0.0
       err <- liftEffect $ liftST $ STRef.read error
       for_ err throwError
 
-      ma <- await
-      case ma of
-        Nothing -> cleanup cancel
-        Just a -> do
-          res <- liftEffect $ O.write w a
-          case res of
-            O.WriteOk -> pure $ Loop { error, cancel }
-            O.WriteWouldBlock -> do
-              liftAff (O.awaitWritableOrClosed w)
-              pure $ Loop { error, cancel }
-            O.WriteClosed -> cleanup cancel
+      needsDrain <- liftEffect $ O.needsDrain w
+      when needsDrain $ liftAff $ O.awaitWritableOrClosed w
+
+      ended <- liftEffect $ O.isWritableEnded w
+      if ended then
+        cleanup cancel
+      else
+        await >>= case _ of
+          Nothing -> cleanup cancel
+          Just a -> do
+            void $ liftEffect $ O.write w a
+            pure $ Loop { error, cancel }
   in
     do
       r <- liftEffect $ O.withErrorST w
@@ -82,37 +90,53 @@ fromWritable w =
 -- |
 -- | When `Nothing` is piped to this, the `Transform` stream will
 -- | be `end`ed, and the pipe will noop if invoked again.
-fromTransform :: forall a b. O.Transform a b -> Pipe (Maybe a) (Maybe b) Aff Unit
+fromTransform :: forall a b m. MonadThrow Error m => MonadAff m => O.Transform a b -> Pipe (Maybe a) (Maybe b) m Unit
 fromTransform t =
   let
     cleanup removeErrorListener = do
       liftEffect $ O.end t
-      liftEffect $ removeErrorListener
+      liftAff $ O.awaitFinished t
       fromReadable t
+      liftEffect $ removeErrorListener
       pure $ Done unit
-    yieldFromReadableHalf = do
-      res <- liftEffect (O.read t)
+
+    yieldWhileReadable = do
+      flip tailRecM unit \_ -> do
+        res <- liftEffect $ O.read t
+        case res of
+          O.ReadJust a -> yield (Just a) $> Loop unit
+          _ -> pure $ Done unit
+
+    maybeYield1 = do
+      res <- liftEffect $ O.read t
       case res of
-        O.ReadJust a -> yield (Just a)
-        O.ReadWouldBlock -> pure unit
-        O.ReadClosed -> yield Nothing *> pure unit
+        O.ReadJust a -> yield $ Just a
+        _ -> pure unit
+
     go { error, cancel } = do
-      liftAff $ delay $ wrap 0.0
       err <- liftEffect $ liftST $ STRef.read error
       for_ err throwError
 
-      ma <- await
-      case ma of
-        Nothing -> cleanup cancel
-        Just a' -> do
-          res <- liftEffect $ O.write t a'
-          yieldFromReadableHalf
-          case res of
-            O.WriteClosed -> cleanup cancel
-            O.WriteOk -> pure $ Loop { error, cancel }
-            O.WriteWouldBlock -> do
-              lift (O.awaitWritableOrClosed t)
-              pure $ Loop { error, cancel }
+      needsDrain <- liftEffect $ O.needsDrain t
+      ended <- liftEffect $ O.isWritableEnded t
+      if needsDrain then do
+        liftAff $ delay $ wrap 0.0
+        yieldWhileReadable
+        pure $ Loop { error, cancel }
+      else if ended then
+        cleanup cancel
+      else
+        await >>= case _ of
+          Nothing -> cleanup cancel
+          Just a' -> do
+            res <- liftEffect $ O.write t a'
+            case res of
+              O.WriteOk -> do
+                maybeYield1
+                pure $ Loop { error, cancel }
+              O.WriteWouldBlock -> do
+                yieldWhileReadable
+                pure $ Loop { error, cancel }
   in
     do
       r <- liftEffect $ O.withErrorST t
@@ -121,13 +145,13 @@ fromTransform t =
 -- | Given a `Producer` of values, wrap them in `Just`.
 -- |
 -- | Before the `Producer` exits, emits `Nothing` as an End-of-stream signal.
-withEOS :: forall a. Producer a Aff Unit -> Producer (Maybe a) Aff Unit
+withEOS :: forall a m. Monad m => Producer a m Unit -> Producer (Maybe a) m Unit
 withEOS a = do
   P.for a (yield <<< Just)
   yield Nothing
 
 -- | Strip a pipeline of the EOS signal
-unEOS :: forall a. Pipe (Maybe a) a Aff Unit
+unEOS :: forall a m. Monad m => Pipe (Maybe a) a m Unit
 unEOS = P.mapFoldable identity
 
 -- | Lift a `Pipe a a` to `Pipe (Maybe a) (Maybe a)`.
@@ -140,8 +164,16 @@ unEOS = P.mapFoldable identity
 -- | `Just` values will be passed to the pipe, and the response(s) will be wrapped in `Just`.
 -- |
 -- | `Nothing` will bypass the given pipe entirely, and the pipe will not be invoked again.
-inEOS :: forall a b. Pipe a b Aff Unit -> Pipe (Maybe a) (Maybe b) Aff Unit
-inEOS p = whileJust do
+inEOS :: forall a b m. MonadRec m => Pipe a b m Unit -> Pipe (Maybe a) (Maybe b) m Unit
+inEOS p = flip tailRecM p \p' -> do
   ma <- await
-  maybe (yield Nothing) (\a -> yield a >-> p >-> P.map Just) ma
-  pure $ void ma
+  case ma of
+    Just a -> do
+      res <- lift $ invoke p' a
+      case res of
+        Yielded (as /\ p'') -> do
+          for_ (Just <$> as) yield
+          pure $ Loop p''
+        DidNotYield p'' -> pure $ Loop p''
+        Exited -> yield Nothing $> Done unit
+    _ -> yield Nothing $> Done unit

src/Pipes.Node.Zlib.purs

@@ -2,10 +2,12 @@ module Pipes.Node.Zlib where
 
 import Prelude
 
+import Control.Monad.Error.Class (class MonadThrow)
 import Data.Maybe (Maybe)
 import Effect (Effect)
-import Effect.Aff (Aff)
+import Effect.Aff.Class (class MonadAff)
 import Effect.Class (liftEffect)
+import Effect.Exception (Error)
 import Node.Buffer (Buffer)
 import Node.Stream.Object as O
 import Node.Zlib as Zlib
@@ -13,28 +15,28 @@ import Node.Zlib.Types (ZlibStream)
 import Pipes.Core (Pipe)
 import Pipes.Node.Stream (fromTransform)
 
-fromZlib :: forall r. Effect (ZlibStream r) -> Pipe (Maybe Buffer) (Maybe Buffer) Aff Unit
+fromZlib :: forall r m. MonadAff m => MonadThrow Error m => Effect (ZlibStream r) -> Pipe (Maybe Buffer) (Maybe Buffer) m Unit
 fromZlib z = do
   raw <- liftEffect $ Zlib.toDuplex <$> z
   fromTransform $ O.fromBufferTransform raw
 
-gzip :: Pipe (Maybe Buffer) (Maybe Buffer) Aff Unit
+gzip :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
 gzip = fromZlib Zlib.createGzip
 
-gunzip :: Pipe (Maybe Buffer) (Maybe Buffer) Aff Unit
+gunzip :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
 gunzip = fromZlib Zlib.createGunzip
 
-unzip :: Pipe (Maybe Buffer) (Maybe Buffer) Aff Unit
+unzip :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
 unzip = fromZlib Zlib.createUnzip
 
-inflate :: Pipe (Maybe Buffer) (Maybe Buffer) Aff Unit
+inflate :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
 inflate = fromZlib Zlib.createInflate
 
-deflate :: Pipe (Maybe Buffer) (Maybe Buffer) Aff Unit
+deflate :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
 deflate = fromZlib Zlib.createDeflate
 
-brotliCompress :: Pipe (Maybe Buffer) (Maybe Buffer) Aff Unit
+brotliCompress :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
 brotliCompress = fromZlib Zlib.createBrotliCompress
 
-brotliDecompress :: Pipe (Maybe Buffer) (Maybe Buffer) Aff Unit
+brotliDecompress :: forall m. MonadAff m => MonadThrow Error m => Pipe (Maybe Buffer) (Maybe Buffer) m Unit
 brotliDecompress = fromZlib Zlib.createBrotliDecompress

src/Pipes.Util.purs

@@ -3,17 +3,34 @@ module Pipes.Util where
 import Prelude
 
 import Control.Monad.Maybe.Trans (MaybeT(..), runMaybeT)
-import Control.Monad.Rec.Class (whileJust)
+import Control.Monad.Rec.Class (class MonadRec, Step(..), forever, tailRecM)
+import Control.Monad.Rec.Class as Rec
 import Control.Monad.ST.Class (liftST)
 import Control.Monad.ST.Ref (STRef)
 import Control.Monad.ST.Ref as STRef
 import Control.Monad.Trans.Class (lift)
 import Data.Array.ST (STArray)
 import Data.Array.ST as Array.ST
-import Data.Maybe (Maybe(..))
+import Data.Either (hush)
+import Data.HashSet as HashSet
+import Data.Hashable (class Hashable, hash)
+import Data.List.NonEmpty (NonEmptyList)
+import Data.Maybe (Maybe(..), fromMaybe)
+import Data.Tuple.Nested (type (/\), (/\))
 import Effect.Class (class MonadEffect, liftEffect)
 import Pipes (await, yield)
-import Pipes.Core (Pipe)
+import Pipes as Pipes
+import Pipes.Core (Pipe, Producer)
+import Pipes.Internal (Proxy(..))
+
+-- | Re-yield all `Just`s, and close when `Nothing` is encountered
+whileJust :: forall m a. MonadRec m => Pipe (Maybe a) a m Unit
+whileJust = do
+  first <- await
+  flip tailRecM first $ \ma -> fromMaybe (Done unit) <$> runMaybeT do
+    a <- MaybeT $ pure ma
+    lift $ yield a
+    lift $ Loop <$> await
 
 -- | Yields a separator value `sep` between received values
 -- |
@@ -28,7 +45,7 @@ intersperse sep = do
     getIsFirst = liftEffect $ liftST $ STRef.read isFirstST
     markNotFirst = void $ liftEffect $ liftST $ STRef.write false isFirstST
 
-  whileJust $ runMaybeT do
+  Rec.whileJust $ runMaybeT do
     a <- MaybeT await
     isFirst <- getIsFirst
     if isFirst then markNotFirst else lift $ yield $ Just sep
@@ -36,6 +53,16 @@ intersperse sep = do
 
   yield Nothing
 
+-- Pair every emitted value from 2 producers together, exiting when either exits.
+zip :: forall a b m. MonadRec m => Producer a m Unit -> Producer b m Unit -> Producer (a /\ b) m Unit
+zip as bs =
+  flip tailRecM (as /\ bs) \(as' /\ bs') ->
+    fromMaybe (Done unit) <$> runMaybeT do
+      a /\ as'' <- MaybeT $ lift $ hush <$> Pipes.next as'
+      b /\ bs'' <- MaybeT $ lift $ hush <$> Pipes.next bs'
+      lift $ yield $ a /\ b
+      pure $ Loop $ as'' /\ bs''
+
 -- | Accumulate values in chunks of a given size.
 -- |
 -- | If the pipe closes without yielding a multiple of `size` elements,
@@ -55,10 +82,65 @@ chunked size = do
       void $ flip STRef.write chunkST =<< Array.ST.new
       Array.ST.unsafeFreeze chunkArray
 
-  whileJust $ runMaybeT do
+  Rec.whileJust $ runMaybeT do
     a <- MaybeT await
     chunkPut a
-    len <- chunkLength
+    len <- lift chunkLength
     when (len >= size) $ lift $ yield =<< Just <$> chunkTake
-  yield =<< Just <$> chunkTake
+  len <- chunkLength
+  when (len > 0) $ yield =<< Just <$> chunkTake
   yield Nothing
+
+-- | Equivalent of unix `uniq`, filtering out duplicate values passed to it.
+-- |
+-- | Uses a `HashSet` of hashes of `a`; for `n` elements `awaited`, this pipe
+-- | will occupy O(n) space, and `yield` in O(1) time.
+uniqHash :: forall a m. Hashable a => MonadEffect m => MonadRec m => Pipe a a m Unit
+uniqHash = do
+  seenHashesST <- liftEffect $ liftST $ STRef.new HashSet.empty
+  forever do
+    a <- await
+    seenHashes <- liftEffect $ liftST $ STRef.read seenHashesST
+    when (not $ HashSet.member (hash a) seenHashes) do
+      void $ liftEffect $ liftST $ STRef.modify (HashSet.insert $ hash a) seenHashesST
+      yield a
+
+-- | The result of a single step forward of a pipe.
+data InvokeResult a b m
+  -- | The pipe `await`ed the value, but did not `yield` a response.
+  = DidNotYield (Pipe a b m Unit)
+  -- | The pipe `await`ed the value, and `yield`ed 1 or more responses.
+  | Yielded (NonEmptyList b /\ Pipe a b m Unit)
+  -- | The pipe `await`ed the value, and exited.
+  | Exited
+
+data IntermediateInvokeResult a b m
+  = IDidNotYield (Pipe a b m Unit)
+  | IYielded (NonEmptyList b /\ Pipe a b m Unit)
+  | IDidNotAwait (Pipe a b m Unit)
+
+-- | Pass a single value to a pipe, returning the result of the pipe's invocation.
+invoke :: forall m a b. Monad m => Pipe a b m Unit -> a -> m (InvokeResult a b m)
+invoke m a =
+  let
+    go :: IntermediateInvokeResult a b m -> m (InvokeResult a b m)
+    go (IYielded (as /\ n)) =
+      case n of
+        Request _ _ -> pure $ Yielded $ as /\ n
+        Respond rep f -> go (IYielded $ (as <> pure rep) /\ f unit)
+        M o -> go =<< IYielded <$> (as /\ _) <$> o
+        Pure _ -> pure Exited
+    go (IDidNotYield n) =
+      case n of
+        Request _ _ -> pure $ DidNotYield n
+        Respond rep f -> go (IYielded $ pure rep /\ f unit)
+        M o -> go =<< IDidNotYield <$> o
+        Pure _ -> pure Exited
+    go (IDidNotAwait n) =
+      case n of
+        Request _ f -> go (IDidNotYield (f a))
+        Respond rep f -> go (IYielded $ pure rep /\ f unit)
+        M o -> go =<< IDidNotAwait <$> o
+        Pure _ -> pure Exited
+  in
+    go (IDidNotAwait m)

test/Test/Main.purs

@@ -9,12 +9,14 @@ import Test.Pipes.Node.Stream as Test.Pipes.Node.Stream
 import Test.Pipes.Node.Buffer as Test.Pipes.Node.Buffer
 import Test.Pipes.Node.FS as Test.Pipes.Node.FS
 import Test.Pipes.Collect as Test.Pipes.Collect
+import Test.Pipes.Construct as Test.Pipes.Construct
 import Test.Spec.Reporter (specReporter)
 import Test.Spec.Runner (defaultConfig, runSpec')
 
 main :: Effect Unit
-main = launchAff_ $ runSpec' (defaultConfig { failFast = true, timeout = Nothing }) [ specReporter ] do
+main = launchAff_ $ runSpec' (defaultConfig { exit = false, timeout = Nothing }) [ specReporter ] do
   Test.Pipes.Node.Stream.spec
   Test.Pipes.Node.Buffer.spec
   Test.Pipes.Node.FS.spec
   Test.Pipes.Collect.spec
+  Test.Pipes.Construct.spec

test/Test/Pipes.Construct.purs

@@ -0,0 +1,58 @@
+module Test.Pipes.Construct where
+
+import Prelude
+
+import Data.Array as Array
+import Data.List as List
+import Data.Map as Map
+import Data.Tuple.Nested (type (/\), (/\))
+import Effect.Class (liftEffect)
+import Pipes.Collect as Pipes.Collect
+import Pipes.Construct as Pipes.Construct
+import Test.Spec (Spec, describe, it)
+import Test.Spec.Assertions (shouldEqual)
+
+spec :: Spec Unit
+spec =
+  describe "Test.Pipes.Construct" do
+    describe "eachMap" do
+      it "empty map" do
+        kvs <- Pipes.Collect.toArray $ Pipes.Construct.eachMap Map.empty
+        kvs `shouldEqual` ([] :: Array (Int /\ Int))
+      it "nonempty map" do
+        let
+          exp = (\n -> n /\ n) <$> Array.range 0 99999
+          map = Map.fromFoldable exp
+        kvs <-
+          liftEffect
+            $ Pipes.Collect.toArray
+            $ Pipes.Construct.eachMap
+            $ map
+        kvs `shouldEqual` exp
+    describe "eachArray" do
+      it "empty array" do
+        kvs <- Pipes.Collect.toArray $ Pipes.Construct.eachArray []
+        kvs `shouldEqual` ([] :: Array Int)
+      it "nonempty array" do
+        let
+          inp = (\n -> n /\ n) <$> Array.range 0 99999
+        kvs <-
+          liftEffect
+            $ Pipes.Collect.toArray
+            $ Pipes.Construct.eachArray
+            $ inp
+        kvs `shouldEqual` inp
+    describe "eachList" do
+      it "empty list" do
+        kvs <- Pipes.Collect.toArray $ Pipes.Construct.eachList List.Nil
+        kvs `shouldEqual` ([] :: Array Int)
+      it "nonempty list" do
+        let
+          inp = (\n -> n /\ n) <$> Array.range 0 99999
+        kvs <-
+          liftEffect
+            $ Pipes.Collect.toArray
+            $ Pipes.Construct.eachList
+            $ List.fromFoldable
+            $ inp
+        kvs `shouldEqual` inp

test/Test/Pipes.Node.FS.purs

@@ -34,7 +34,7 @@ spec = describe "Pipes.Node.FS" do
       s <- fold <$> Pipes.toListM (Pipes.Node.FS.read p >-> unEOS >-> Pipes.Node.Buffer.toString UTF8)
       s `shouldEqual` "foo"
     around tmpFile $ it "fails if the file already exists" \p -> do
-      liftEffect $ FS.writeTextFile UTF8 "foo" p
+      liftEffect $ FS.writeTextFile UTF8 p "foo"
       flip catchError (const $ pure unit) do
         Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.create p
         fail "should have thrown"
@@ -44,7 +44,7 @@ spec = describe "Pipes.Node.FS" do
       contents <- liftEffect $ FS.readTextFile UTF8 p
       contents `shouldEqual` "foo"
     around tmpFile $ it "fails if the file already exists" \p -> do
-      liftEffect $ FS.writeTextFile UTF8 "foo" p
+      liftEffect $ FS.writeTextFile UTF8 p "foo"
       flip catchError (const $ pure unit) do
         Pipes.runEffect $ withEOS (yield "foo" >-> Pipes.Node.Buffer.fromString UTF8) >-> Pipes.Node.FS.create p
         fail "should have thrown"