orion/purescript-csv-stream
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: generalize to MonadAff

Browse Source
This commit is contained in:
orion kindel
2024-05-13 11:52:09 -05:00
parent b4b6dfdebd
commit e87d81cc1d
5 changed files with 72 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
test/Test/Pipes.CSV.purs
View File

@@ -14,7 +14,6 @@ import Effect.Class (liftEffect)
import Node.Encoding (Encoding(..))
import Partial.Unsafe (unsafePartial)
import Pipes (yield, (>->))
import Pipes (each) as Pipes
import Pipes.CSV as Pipes.CSV
import Pipes.Collect as Pipes.Collect
import Pipes.Node.Buffer as Pipes.Buffer
@@ -26,7 +25,8 @@ import Test.Spec (Spec, describe, it)
import Test.Spec.Assertions (shouldEqual)
csv :: String
csv = """created,flag,foo,id
csv =
"""created,flag,foo,id
2020-01-01T00:00:00.0Z,true,a,1
2024-02-02T08:00:00.0Z,false,apple,2
1970-01-01T00:00:00.0Z,true,hello,3
@@ -41,31 +41,31 @@ spec =
it "stringify" do
let
objs =
[ {id: 1, foo: "a", flag: true, created: dt "2020-01-01T00:00:00Z"}
, {id: 2, foo: "apple", flag: false, created: dt "2024-02-02T08:00:00Z"}
, {id: 3, foo: "hello", flag: true, created: dt "1970-01-01T00:00:00Z"}
[ { id: 1, foo: "a", flag: true, created: dt "2020-01-01T00:00:00Z" }
, { id: 2, foo: "apple", flag: false, created: dt "2024-02-02T08:00:00Z" }
, { id: 3, foo: "hello", flag: true, created: dt "1970-01-01T00:00:00Z" }
]
csv' <- map fold $ Pipes.Collect.collectArray $ Pipes.Stream.withEOS (Pipes.each objs) >-> Pipes.CSV.stringify >-> Pipes.Stream.unEOS
csv' <- map fold $ Pipes.Collect.toArray $ Pipes.Stream.withEOS (Pipes.Construct.eachArray objs) >-> Pipes.CSV.stringify >-> Pipes.Stream.unEOS
csv' `shouldEqual` csv
describe "parse" do
it "parses csv" do
rows <- map Array.fromFoldable
$ Pipes.toListM
$ Pipes.Stream.withEOS (yield csv)
>-> Pipes.Stream.inEOS (Pipes.Buffer.fromString UTF8)
>-> Pipes.CSV.parse
>-> Pipes.Stream.unEOS
>-> Pipes.Stream.inEOS (Pipes.Buffer.fromString UTF8)
>-> Pipes.CSV.parse
>-> Pipes.Stream.unEOS
rows `shouldEqual`
[ {id: 1, foo: "a", flag: true, created: dt "2020-01-01T00:00:00Z"}
, {id: 2, foo: "apple", flag: false, created: dt "2024-02-02T08:00:00Z"}
, {id: 3, foo: "hello", flag: true, created: dt "1970-01-01T00:00:00Z"}
[ { id: 1, foo: "a", flag: true, created: dt "2020-01-01T00:00:00Z" }
, { id: 2, foo: "apple", flag: false, created: dt "2024-02-02T08:00:00Z" }
, { id: 3, foo: "hello", flag: true, created: dt "1970-01-01T00:00:00Z" }
]
it "parses large csv" do
nums <- liftEffect $ randomSample' 100000 (chooseInt 0 9)
let
csvRows = ["id\n"] <> ((_ <> "\n") <$> show <$> nums)
csvRows = [ "id\n" ] <> ((_ <> "\n") <$> show <$> nums)
csv' =
let
go ix
@@ -75,14 +75,14 @@ spec =
tailRecM go 0
in16kbChunks =
Pipes.Util.chunked 16000
>-> Pipes.Stream.inEOS (Pipes.map fold)
>-> Pipes.Stream.inEOS (Pipes.Buffer.fromString UTF8)
>-> Pipes.Stream.inEOS (Pipes.map fold)
>-> Pipes.Stream.inEOS (Pipes.Buffer.fromString UTF8)
rows <-
Pipes.Collect.collectArray
$ Pipes.Stream.withEOS csv'
>-> in16kbChunks
>-> Pipes.CSV.parse
>-> Pipes.Stream.unEOS
Pipes.Collect.toArray
$ Pipes.Stream.withEOS csv'
>-> in16kbChunks
>-> Pipes.CSV.parse
>-> Pipes.Stream.unEOS
rows `shouldEqual` ((\id -> {id}) <$> nums)
rows `shouldEqual` ((\id -> { id }) <$> nums)