diff --git a/.github/workflows/haskell.yml b/.github/workflows/haskell.yml index 1466765298..702fb14368 100644 --- a/.github/workflows/haskell.yml +++ b/.github/workflows/haskell.yml @@ -194,7 +194,7 @@ jobs: # This includes the debug flag as well - name: dev-no-opt - runner: macos-latest + runner: ubuntu-latest command: cabal ghc_version: 9.14.1 # WARNING! cannot use # comments inside pack_options. @@ -246,9 +246,11 @@ jobs: runner: ubuntu-latest command: cabal ghc_version: 9.8.4 + # The deprecated tests need the deprecated modules of streamly. # WARNING! cannot use # comments inside pack_options. pack_options: >- CABAL_PROJECT=cabal.project + CABAL_BUILD_OPTIONS="--flag deprecated-tests --constraint=streamly+internal-dev" ignore_error: false - name: streamly-sdist @@ -413,7 +415,7 @@ jobs: with: path: | ~/.cache/cabal/packages - key: cabal-hackage-index-non-windows + key: cabal-hackage-index # GHC is not test/options specific - name: Restore ghcup and ghc (non-Windows) @@ -554,7 +556,7 @@ jobs: if: always() && steps.deps-only.outcome == 'success' && runner.os != 'Windows' && steps.restore-hackage.outputs.cache-hit != 'true' with: path: ~/.cache/cabal/packages - key: cabal-hackage-index-non-windows + key: cabal-hackage-index - name: Save ghcup and ghc (non-Windows) uses: actions/cache/save@v4 diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml index 9a3d927dc5..02b9be05ed 100644 --- a/.github/workflows/js.yml +++ b/.github/workflows/js.yml @@ -97,14 +97,25 @@ jobs: cabal build --project-file=cabal.project.ghcjs \ --builddir=$BUILDDIR all + # Run the test executables built above instead of using "cabal test". + # GHC recompiles the modules that use Template Haskell on every cabal + # invocation, "cabal test" would spend about 5 minutes rebuilding them. + # Like "cabal test", run each test in the package directory. - name: Test run: | - set +e - cabal test --keep-going --project-file=cabal.project.ghcjs \ - --builddir=$BUILDDIR all 2>&1 | tee test.log - set -e - grep -E "^Test suite .*: (PASS|FAIL)$" test.log - failed=$(sed -n 's/^Test suite \(.*\): FAIL$/\1/p' test.log) + tests=$(jq -r '."install-plan"[] + | select(."pkg-name" == "streamly-tests" + and ((."component-name" // "") | startswith("test:"))) + | ."component-name" + " " + ."bin-file"' $BUILDDIR/cache/plan.json) + failed= + cd test + while read -r comp bin; do + t=${comp#test:} + echo "::group::$t" + if "$bin" < /dev/null; then r=PASS; else r=FAIL; failed="$failed $t"; fi + echo "::endgroup::" + echo "Test suite $t: $r" + done <<< "$tests" unexpected= for t in $failed; do case " $KNOWN_FAILURES " in @@ -116,8 +127,3 @@ jobs: echo "Unexpected failures:$unexpected" exit 1 fi - # A test suite that fails to build is not reported as FAIL - if grep -q "^Error: \[Cabal-" test.log; then - echo "cabal test failed to build some test suites" - exit 1 - fi diff --git a/.github/workflows/markdown-doctest.yml b/.github/workflows/markdown-doctest.yml index f55e236e10..7400a0e215 100644 --- a/.github/workflows/markdown-doctest.yml +++ b/.github/workflows/markdown-doctest.yml @@ -20,22 +20,21 @@ jobs: run: | rm -f ~/.ghcup - - name: Cache ghcup and ghc (non-Windows) - uses: actions/cache@v4 - if: runner.os != 'Windows' + # Saved by the TEST workflow on master + - name: Restore ghcup and ghc + uses: actions/cache/restore@v4 with: path: | ~/.ghcup - key: ${{ runner.os }}-ghcup-${{ env.GHCUP_VERSION }}-${{ env.GHCVER }} + key: ghc-${{ env.GHCVER }}-v-ubuntu-latest - - name: Cache hackage package index (non-Windows) - uses: actions/cache@v4 - if: runner.os != 'Windows' + # Saved by the TEST workflow on master, "cabal update" updates it. + - name: Restore hackage package index + uses: actions/cache/restore@v4 with: path: | ~/.cache/cabal/packages - # Bump the key version to clear the cache - key: cache-cabal-packages + key: cabal-hackage-index # NOTE: we need the cabal state as well because the executable may depend # on shared libraries in the cabal store. @@ -59,7 +58,7 @@ jobs: - name: Download ghc run: | - if ! ghc --version 2>/dev/null | grep -q "$GHCVER" + if ! ghc --version 2>/dev/null | grep -q "$GHCVER" || ! command -v cabal then curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VERSION/x86_64-linux-ghcup-$GHCUP_VERSION chmod +x ./ghcup diff --git a/.github/workflows/packdiff.yml b/.github/workflows/packdiff.yml index 0962df1e23..1f6aab5142 100644 --- a/.github/workflows/packdiff.yml +++ b/.github/workflows/packdiff.yml @@ -20,22 +20,21 @@ jobs: run: | rm -f ~/.ghcup - - name: Cache ghcup and ghc (non-Windows) - uses: actions/cache@v4 - if: runner.os != 'Windows' + # Saved by the TEST workflow on master + - name: Restore ghcup and ghc + uses: actions/cache/restore@v4 with: path: | ~/.ghcup - key: ${{ runner.os }}-ghcup-${{ env.GHCUP_VERSION }}-${{ env.GHCVER }} + key: ghc-${{ env.GHCVER }}-v-ubuntu-latest - - name: Cache hackage package index (non-Windows) - uses: actions/cache@v4 - if: runner.os != 'Windows' + # Saved by the TEST workflow on master, "cabal update" updates it. + - name: Restore hackage package index + uses: actions/cache/restore@v4 with: path: | ~/.cache/cabal/packages - # Bump the key version to clear the cache - key: cache-cabal-packages + key: cabal-hackage-index - name: Cache packdiff (non-Windows) uses: actions/cache@v4 @@ -59,7 +58,7 @@ jobs: - name: Download ghc run: | - if ! ghc --version 2>/dev/null | grep -q "$GHCVER" + if ! ghc --version 2>/dev/null | grep -q "$GHCVER" || ! command -v cabal then curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VERSION/x86_64-linux-ghcup-$GHCUP_VERSION chmod +x ./ghcup diff --git a/.github/workflows/regression-check.yml b/.github/workflows/regression-check.yml index 110ae88913..036dfe9ee8 100644 --- a/.github/workflows/regression-check.yml +++ b/.github/workflows/regression-check.yml @@ -3,6 +3,11 @@ name: PERF on: workflow_dispatch: pull_request: + # On master only the benchmarks are built and cached, so that pull requests + # can reuse them. + push: + branches: + - master # References: # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow @@ -70,31 +75,54 @@ jobs: run: | rm -f ~/.ghcup - - name: Cache ghcup and ghc (non-Windows) - uses: actions/cache@v4 - if: runner.os != 'Windows' + # Saved by the TEST workflow on master + - name: Restore ghcup and ghc + uses: actions/cache/restore@v4 with: path: | ~/.ghcup - key: ${{ runner.os }}-ghcup-${{ env.GHCUP_VERSION }}-${{ env.GHCVER }} + key: ghc-${{ env.GHCVER }}-v-ubuntu-latest - - name: Cache hackage package index (non-Windows) - uses: actions/cache@v4 - if: runner.os != 'Windows' + # Saved by the TEST workflow on master, "cabal update" updates it. + - name: Restore hackage package index + uses: actions/cache/restore@v4 with: path: | ~/.cache/cabal/packages - # Bump the key version to clear the cache - key: cache-cabal-packages + key: cabal-hackage-index - - name: Cache cabal build dependencies (non-Windows) - uses: actions/cache@v4 - if: runner.os != 'Windows' + - name: Determine the "master" commit + run: | + if test "${{ github.event_name }}" = push + then + sha=${{ github.sha }} + else + sha=$(git ls-remote origin refs/heads/master | cut -f1) + fi + echo "MASTER_SHA=$sha" >> $GITHUB_ENV + + # Saved by the push to master run of this workflow. A pull request can + # restore only the caches saved by itself or by master. When master has + # not changed the master benchmarks are up to date and only need to be + # run, otherwise an older master build is restored and the build is + # incremental. The cabal store is in the same cache so that the + # dependencies match the build directory. + - name: Restore the "master" build + id: master-build + uses: actions/cache/restore@v4 with: path: | - ~/.local/state/cabal - # Bump the key version to clear the cache - key: performance-comparison-deps + ~/.local/state/cabal + dist-newstyle + key: perf-master-build-${{ env.GHCVER }}-${{ env.MASTER_SHA }} + restore-keys: perf-master-build-${{ env.GHCVER }}- + + - name: Cache bench-runner + id: bench-runner + uses: actions/cache@v4 + with: + path: bench-runner + key: bench-runner-${{ env.GHCVER }}-${{ hashFiles('cabal.project.report', 'benchmark/bench-runner/**', 'targets/**') }} - name: Update CI_BENCHMARKS environment variable run: | @@ -112,7 +140,7 @@ jobs: - name: Download ghc run: | - if ! ghc --version 2>/dev/null | grep -q "$GHCVER" + if ! ghc --version 2>/dev/null | grep -q "$GHCVER" || ! command -v cabal then curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VERSION/x86_64-linux-ghcup-$GHCUP_VERSION chmod +x ./ghcup @@ -124,9 +152,11 @@ jobs: # XXX update only if not present cabal update + # Use a separate build directory to keep dist-newstyle for the benchmarks - name: Install bench-runner from PR branch + if: steps.bench-runner.outputs.cache-hit != 'true' run: | - cabal install bench-runner --project-file=cabal.project.report --installdir=./ --allow-newer + cabal install bench-runner --project-file=cabal.project.report --builddir=dist-bench-runner --installdir=./ --allow-newer # ----------------------------------------------------------------- # -- Generate reports for the base branch and upload @@ -135,10 +165,33 @@ jobs: - name: Checkout the "master" branch uses: actions/checkout@v4 with: - ref: master + ref: ${{ env.MASTER_SHA }} clean: false + # Only build on master. The build command must be the same as the one + # that bench-runner uses, otherwise the pull request runs rebuild. + - name: Build benchmarks for "master" branch + if: github.event_name == 'push' + run: | + targets= + for b in $CI_BENCHMARKS + do + targets="$targets streamly-benchmarks:bench:$b" + done + cabal v2-build --flag fusion-plugin --flag limit-build-mem \ + --with-compiler ghc --enable-benchmarks $targets + + - name: Save the "master" build + if: github.event_name == 'push' && steps.master-build.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: | + ~/.local/state/cabal + dist-newstyle + key: ${{ steps.master-build.outputs.cache-primary-key }} + - name: Run benchmarks for "master" branch + if: github.event_name != 'push' run: | # --cabal-build-options "-j1" ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw @@ -151,12 +204,14 @@ jobs: # ----------------------------------------------------------------- - name: Checkout the PR branch + if: github.event_name != 'push' uses: actions/checkout@v4 with: # preserve bench-runner and the "charts" directory created by bench-runner clean: false - name: Run benchmarks and append + if: github.event_name != 'push' run: | # --cabal-build-options "-j1" ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw --append @@ -166,10 +221,12 @@ jobs: # ----------------------------------------------------------------- - name: List all benchmarks + if: github.event_name != 'push' run: | ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --no-measure - name: Compare benchmarks + if: github.event_name != 'push' run: | EXIT_STATUS=0 diff --git a/core/docs/Changelog.md b/core/docs/Changelog.md index c24ddb123b..9a4a4d41dd 100644 --- a/core/docs/Changelog.md +++ b/core/docs/Changelog.md @@ -23,9 +23,15 @@ separator, e.g. separator `"abc"` and input `"abcab"`. * Fix `chunksOf` in the array modules emitting a trailing empty array when the stream length is a multiple of the chunk size. -* Fix `Fold.classifyScan`, `classifyScanIO`, `classify` and `classifyIO` - restarting the fold for a key that terminated on its first input. +* Fix `classify` and `classifyIO` in Fold and Scanl, and + `Fold.classifyScan` and `classifyScanIO`, restarting the fold for a key + that terminated on its first input. * Fix the `Arrow` instance of `Scan` failing when a scan skips an input. +* Fix `RingArray.ringsOf` and `scanRingsOf` producing rings of a wrong size + after `2^31` bytes when `Int` is 32-bit, e.g. on the JavaScript backend. +* Fix `Scanl.uniq` and `uniqBy` failing before the first input. +* Fix `Scanl.partition`, `partitionBy` and `partitionByM` not finalizing one + scan when the other terminates. * On POSIX platforms, the `Path` based APIs that open an existing file in `WriteMode`, e.g. `write` and `withFile` in `Streamly.FileSystem.FileIO`, diff --git a/core/src/Streamly/Internal/Data/Fold/Window.hs b/core/src/Streamly/Internal/Data/Fold/Window.hs index c379e439e4..64ceb1c146 100644 --- a/core/src/Streamly/Internal/Data/Fold/Window.hs +++ b/core/src/Streamly/Internal/Data/Fold/Window.hs @@ -301,7 +301,9 @@ windowRange n = Fold step initial extract extract step (Tuple3Fused' mba rh i) a = do RingArray _ _ rh1 <- RingArray.replace_ (RingArray mba (n * SIZE_OF(a)) rh) a - return $ Partial $ Tuple3Fused' mba rh1 (i + 1) + -- i is the number of elements in the window, at most n + let i1 = min (i + 1) n + return $ Partial $ Tuple3Fused' mba rh1 i1 -- XXX exitify optimization causes a problem here when modular folds are -- used. Sometimes inlining "extract" is helpful. @@ -311,8 +313,8 @@ windowRange n = Fold step initial extract extract -- XXX If exiting one was equal to min only then we need to find new min -- XXX We can supply a custom extract function to a generic window -- operation. - let rs = min i n * SIZE_OF(a) - rh1 = if i <= n then 0 else rh + let rs = i * SIZE_OF(a) + rh1 = if i < n then 0 else rh in ringRange $ RingArray mba rs rh1 -- | Find the minimum element in a rolling window. diff --git a/core/src/Streamly/Internal/Data/RingArray.hs b/core/src/Streamly/Internal/Data/RingArray.hs index f81572c5b6..65df61f0d1 100644 --- a/core/src/Streamly/Internal/Data/RingArray.hs +++ b/core/src/Streamly/Internal/Data/RingArray.hs @@ -539,16 +539,25 @@ scanRingsOf n = Scanl step initial extract extract step (Tuple3Fused' mba rh offset) a = do RingArray _ _ rh1 <- replace_ (RingArray mba rSize rh) a - let offset1 = offset + SIZE_OF(a) + -- offset is the size of the ring in bytes, at most rSize + -- + -- XXX GHC 9.14.1 does not inline this step in the fold loop, and + -- boxes the fold state on every input, if we write this as: + -- + -- > if offset < rSize then offset + SIZE_OF(a) else offset + -- + -- Array.createOfLast 1 then allocates 8 MB for 100,000 inputs + -- instead of none. GHC 9.10.3 does not have this problem. Report it + -- to GHC. + let offset1 = min (offset + SIZE_OF(a)) rSize return $ Scanl.Partial $ Tuple3Fused' mba rh1 offset1 -- XXX exitify optimization causes a problem here when modular folds are -- used. Sometimes inlining "extract" is helpful. {-# INLINE extract #-} extract (Tuple3Fused' mba rh offset) = - let rs = min offset rSize - rh1 = if offset <= rSize then 0 else rh - in pure $ RingArray mba rs rh1 + let rh1 = if offset < rSize then 0 else rh + in pure $ RingArray mba offset rh1 -- | @ringsOf n stream@ groups the input stream into a stream of ring arrays of -- size up to n. See 'scanRingsOf' for more details. diff --git a/core/src/Streamly/Internal/Data/Scanl.hs b/core/src/Streamly/Internal/Data/Scanl.hs index cdcda65adf..b68fbddca5 100644 --- a/core/src/Streamly/Internal/Data/Scanl.hs +++ b/core/src/Streamly/Internal/Data/Scanl.hs @@ -16,9 +16,10 @@ -- the output of a scan is a stream. A fold is equivalent to a left scan which -- produces only the final value in the output stream. -- --- Like folds, a scan has an internal state. Unlike a fold, a scan produces an --- output on each input, the output is a function of the scan state and the --- input. +-- Like folds, a scan has an internal state. Unlike a fold, a scan produces at +-- most one output on each input, the output is a function of the scan state +-- and the input. A scan may skip the output on an input using the 'Continue' +-- step, e.g. 'filter'. -- -- A @Scanl m a b@ can represent a @Fold m a b@ by discarding the intermediate -- outputs and keeping only the final output of the scan. @@ -37,7 +38,8 @@ -- == Scanl vs Pipe -- -- A scan is a simpler version of the consumer side of pipes. A left scan --- always produces an output whereas a pipe has an additional ability to skip +-- consumes one input to produce at most one output, whereas a pipe can produce +-- multiple outputs on one input or consume multiple inputs before producing an -- output. Scans are simpler abstractions to think about compared to pipes and -- easier for the compiler to optimize and fuse. -- diff --git a/core/src/Streamly/Internal/Data/Scanl/Combinators.hs b/core/src/Streamly/Internal/Data/Scanl/Combinators.hs index 8843b4b09c..c7cd28cc23 100644 --- a/core/src/Streamly/Internal/Data/Scanl/Combinators.hs +++ b/core/src/Streamly/Internal/Data/Scanl/Combinators.hs @@ -393,39 +393,48 @@ pipe (Pipe consume produce pinitial) (Scanl fstep finitial fextract ffinal) = step (Tuple' cs fs) x = do r <- consume cs x - go fs r + go False fs r where + -- "emitted" is True if the scan returned Partial for an earlier + -- output of the pipe for the same input. If the pipe yields nothing + -- for this input, or the scan returns Continue on all the outputs, + -- we return Continue so that the driver does not emit an output. + next emitted cs1 s = + if emitted + then Partial $ Tuple' cs1 s + else Continue $ Tuple' cs1 s + -- XXX use SPEC? - go acc (Pipe.YieldC cs1 b) = do + go emitted acc (Pipe.YieldC cs1 b) = do acc1 <- fstep acc b return $ case acc1 of Partial s -> Partial $ Tuple' cs1 s - Continue s -> Continue $ Tuple' cs1 s + Continue s -> next emitted cs1 s Done b1 -> Done b1 -- XXX this case is recursive may cause fusion issues. -- To remove recursion we will need a produce mode in scans which makes -- scans similar to pipes except that they do not yield intermediate -- values. - go acc (Pipe.YieldP ps1 b) = do + go emitted acc (Pipe.YieldP ps1 b) = do acc1 <- fstep acc b r <- produce ps1 case acc1 of - Partial s -> go s r - Continue s -> go s r + Partial s -> go True s r + Continue s -> go emitted s r Done b1 -> return $ Done b1 - go acc (Pipe.SkipC cs1) = - return $ Partial $ Tuple' cs1 acc + go emitted acc (Pipe.SkipC cs1) = + return $ next emitted cs1 acc -- XXX this case is recursive may cause fusion issues. - go acc (Pipe.SkipP ps1) = do + go emitted acc (Pipe.SkipP ps1) = do r <- produce ps1 - go acc r + go emitted acc r -- XXX a Stop in consumer means we dropped the input. -- XXX Need to use a "Done b" in pipes as well to represent the same -- behavior as scans. - go acc Pipe.Stop = Done <$> ffinal acc + go _ acc Pipe.Stop = Done <$> ffinal acc extract (Tuple' _ fs) = fextract fs @@ -542,8 +551,11 @@ rollingMapM f = Scanl step initial extract extract where - -- XXX We need just a postscan. We do not need an initial result here. - -- Or we can supply a default initial result as an argument to rollingMapM. + -- TODO: The initial result is an error, so rollingMapM works only as a + -- postscan. Stream.scanl emits the error as its first output, + -- Scanl.compose fails when initialized, and Fold.rollingMap fails on an + -- empty stream. Change the result type to Maybe b, with Nothing before + -- the first input. initial = return $ Fold.Partial (Nothing, error "Empty stream") step (prev, _) cur = do @@ -576,14 +588,20 @@ rollingMap f = rollingMapM (\x y -> return $ f x y) -- {-# INLINE uniqBy #-} uniqBy :: Monad m => (a -> a -> Bool) -> Scanl m a (Maybe a) -uniqBy eq = rollingMap f +uniqBy eq = fmap (\(Tuple' _ x) -> x) $ scanl' step (Tuple' Nothing Nothing) where - f pre curr = - case pre of - Nothing -> Just curr - Just x -> if x `eq` curr then Nothing else Just curr + -- TODO: When rollingMap returns Maybe b, Nothing before the first input, + -- define uniqBy as "fmap join (rollingMap f)", with f the case expression + -- in step. Make the rollingMap state strict (Tuple') first, so that it + -- does not hold the result as a thunk. + step (Tuple' pre _) curr = + let out = + case pre of + Just x | x `eq` curr -> Nothing + _ -> Just curr + in Tuple' (Just curr) out -- | See 'uniqBy'. -- @@ -666,12 +684,12 @@ the = scant' step initial id ------------------------------------------------------------------------------ -- | Determine the sum of all elements of a stream of numbers. Returns additive --- identity (@0@) when the stream is empty. Note that this is not numerically --- stable for floating point numbers. +-- identity (@0@) when the stream is empty. Uses Kahan-Babuska-Neumaier +-- summation, which is numerically stable for floating point numbers. -- -- >>> sum = Scanl.cumulativeScan Scanl.incrSum -- --- Same as following but numerically stable: +-- Same as the following, except that these are not numerically stable: -- -- >>> sum = Scanl.scanl' (+) 0 -- >>> sum = fmap Data.Monoid.getSum $ Scanl.foldMap Data.Monoid.Sum @@ -1776,13 +1794,15 @@ partitionByM f initial = do resL <- initialL resR <- initialR - return - $ case resL of - Fold.Done bl -> Fold.Done bl - Fold.Partial sl -> - case resR of - Fold.Partial sr -> Fold.Partial $ PartLeft sl sr - Fold.Done br -> Fold.Done br + case resL of + Fold.Done bl -> + case resR of + Fold.Partial sr -> finalR sr >> return (Fold.Done bl) + Fold.Done _ -> return $ Fold.Done bl + Fold.Partial sl -> + case resR of + Fold.Partial sr -> return $ Fold.Partial $ PartLeft sl sr + Fold.Done br -> finalL sl >> return (Fold.Done br) runBoth sL sR a = do pRes <- f a @@ -1792,13 +1812,13 @@ partitionByM f case resL of Partial s -> return $ Partial $ PartLeft s sR Continue s -> return $ Continue $ PartLeft s sR - Done x -> return $ Done x + Done x -> finalR sR >> return (Done x) Right c -> do resR <- stepR sR c case resR of Partial s -> return $ Partial $ PartRight sL s Continue s -> return $ Continue $ PartRight sL s - Done x -> return $ Done x + Done x -> finalL sL >> return (Done x) step (PartLeft sL sR) = runBoth sL sR step (PartRight sL sR) = runBoth sL sR @@ -1985,7 +2005,9 @@ zipStreamWithM = undefined zipStream :: Monad m => Stream m a -> Scanl m (a, b) x -> Scanl m b x zipStream = zipStreamWithM (curry return) --- | Pair each element of a scan input with its index, starting from index 0. +-- | @indexingWith i f@ pairs each element of a scan input with an index. The +-- first element gets the index @i@, the index of each following element is +-- @f@ applied to the previous index. -- {-# INLINE indexingWith #-} indexingWith :: Monad m => Int -> (Int -> Int) -> Scanl m a (Maybe (Int, a)) @@ -2048,7 +2070,7 @@ with f comb g = f . comb g . lmap snd {-# INLINE sampleFromthen #-} sampleFromthen :: Monad m => Int -> Int -> Scanl m a b -> Scanl m a b sampleFromthen offset size = - with indexed filter (\(i, _) -> (i + offset) `mod` size == 0) + with indexed filter (\(i, _) -> i >= offset && (i - offset) `mod` size == 0) ------------------------------------------------------------------------------ -- Nesting @@ -2116,6 +2138,9 @@ toStreamRev = fmap StreamD.fromList toListRev -- | Get the bottom most @n@ elements using the supplied comparison function. -- +-- The returned 'MutArray' is the buffer of the scan, later inputs modify it in +-- place. Convert or copy each output before the next input if it is retained. +-- {-# INLINE bottomBy #-} bottomBy :: (MonadIO m, Unbox a) => (a -> a -> Ordering) @@ -2157,8 +2182,11 @@ bottomBy cmp n = Scanl step initial extract extract -- Example: -- -- >>> stream = Stream.fromList [2::Int,7,9,3,1,5,6,11,17] --- >>> Stream.toList (Stream.scanl (Scanl.topBy compare 3) stream) >>= mapM MutArray.toList --- [[],[17],[17,11],[17,11,9],[17,11,9],[17,11,9],[17,11,9],[17,11,9],[17,11,9],[17,11,9]] +-- >>> Stream.toList $ Stream.scanl (Scanl.rmapM MutArray.toList (Scanl.topBy compare 3)) stream +-- [[],[2],[7,2],[9,7,2],[9,7,3],[9,7,3],[9,7,5],[9,7,6],[11,9,7],[17,11,9]] +-- +-- The returned 'MutArray' is the buffer of the scan, later inputs modify it in +-- place. Convert or copy each output before the next input if it is retained. -- -- /Pre-release/ -- @@ -2176,8 +2204,8 @@ topBy cmp = bottomBy (flip cmp) -- >>> top = Scanl.topBy compare -- -- >>> stream = Stream.fromList [2::Int,7,9,3,1,5,6,11,17] --- >>> Stream.toList (Stream.scanl (Scanl.top 3) stream) >>= mapM MutArray.toList --- [[],[17],[17,11],[17,11,9],[17,11,9],[17,11,9],[17,11,9],[17,11,9],[17,11,9],[17,11,9]] +-- >>> Stream.toList $ Stream.scanl (Scanl.rmapM MutArray.toList (Scanl.top 3)) stream +-- [[],[2],[7,2],[9,7,2],[9,7,3],[9,7,3],[9,7,5],[9,7,6],[11,9,7],[17,11,9]] -- -- /Pre-release/ {-# INLINE top #-} @@ -2191,8 +2219,8 @@ top = bottomBy $ flip compare -- >>> bottom = Scanl.bottomBy compare -- -- >>> stream = Stream.fromList [2::Int,7,9,3,1,5,6,11,17] --- >>> Stream.toList (Stream.scanl (Scanl.bottom 3) stream) >>= mapM MutArray.toList --- [[],[1],[1,2],[1,2,3],[1,2,3],[1,2,3],[1,2,3],[1,2,3],[1,2,3],[1,2,3]] +-- >>> Stream.toList $ Stream.scanl (Scanl.rmapM MutArray.toList (Scanl.bottom 3)) stream +-- [[],[2],[2,7],[2,7,9],[2,3,7],[1,2,3],[1,2,3],[1,2,3],[1,2,3],[1,2,3]] -- -- /Pre-release/ {-# INLINE bottom #-} diff --git a/core/src/Streamly/Internal/Data/Scanl/Container.hs b/core/src/Streamly/Internal/Data/Scanl/Container.hs index a635de5362..f619775ffe 100644 --- a/core/src/Streamly/Internal/Data/Scanl/Container.hs +++ b/core/src/Streamly/Internal/Data/Scanl/Container.hs @@ -618,7 +618,7 @@ classifyGeneric f (Scanl step1 initial1 extract1 final1) = return $ Tuple3' (IsMap.mapInsert k s1 kv) set Nothing Done b -> - return $ Tuple3' kv set (Just (k, b)) + return $ Tuple3' kv (Set.insert k set) (Just (k, b)) Fold.Done b -> return (Tuple3' kv (Set.insert k set) (Just (k, b))) step (Tuple3' kv set _) a = do @@ -710,7 +710,7 @@ classifyGenericIO f (Scanl step1 initial1 extract1 final1) = $ Tuple3' (IsMap.mapInsert k ref kv) set Nothing Done b -> - return $ Tuple3' kv set (Just (k, b)) + return $ Tuple3' kv (Set.insert k set) (Just (k, b)) Fold.Done b -> return (Tuple3' kv (Set.insert k set) (Just (k, b))) step (Tuple3' kv set _) a = do diff --git a/core/src/Streamly/Internal/Data/Scanl/Type.hs b/core/src/Streamly/Internal/Data/Scanl/Type.hs index c38b9c44a2..68a516e567 100644 --- a/core/src/Streamly/Internal/Data/Scanl/Type.hs +++ b/core/src/Streamly/Internal/Data/Scanl/Type.hs @@ -687,8 +687,7 @@ minBy cmp x y = minimumBy :: Monad m => (a -> a -> Ordering) -> Scanl m a (Maybe a) minimumBy cmp = scanl1' (minBy cmp) --- | Determine the minimum element in a stream using the supplied comparison --- function. +-- | Determine the minimum element in a stream. -- -- Definitions: -- @@ -696,9 +695,9 @@ minimumBy cmp = scanl1' (minBy cmp) -- >>> minimum = Scanl.scanl1' min -- -- Same as the following but without a default minimum. The 'Min' Monoid uses the --- 'maxBound' as the default maximum: +-- 'maxBound' as the default minimum: -- --- >>> maximum = fmap Data.Semigroup.getMin $ Scanl.foldMap Data.Semigroup.Min +-- >>> minimum = fmap Data.Semigroup.getMin $ Scanl.foldMap Data.Semigroup.Min -- {-# INLINE minimum #-} minimum :: (Monad m, Ord a) => Scanl m a (Maybe a) diff --git a/core/src/Streamly/Internal/Data/Scanl/Window.hs b/core/src/Streamly/Internal/Data/Scanl/Window.hs index 423a92f03a..2a17d1d555 100644 --- a/core/src/Streamly/Internal/Data/Scanl/Window.hs +++ b/core/src/Streamly/Internal/Data/Scanl/Window.hs @@ -453,14 +453,16 @@ windowRange n = Scanl step initial extract extract initial = if n <= 0 - then error "ringsOf: window size must be > 0" + then error "windowRange: window size must be > 0" else do arr :: MutArray.MutArray a <- liftIO $ MutArray.emptyOf n return $ Fold.Partial $ Tuple3Fused' (MutArray.arrContents arr) 0 0 step (Tuple3Fused' mba rh i) a = do RingArray _ _ rh1 <- RingArray.replace_ (RingArray mba (n * SIZE_OF(a)) rh) a - return $ Partial $ Tuple3Fused' mba rh1 (i + 1) + -- i is the number of elements in the window, at most n + let i1 = min (i + 1) n + return $ Partial $ Tuple3Fused' mba rh1 i1 -- XXX exitify optimization causes a problem here when modular scans are -- used. Sometimes inlining "extract" is helpful. @@ -469,8 +471,8 @@ windowRange n = Scanl step initial extract extract -- XXX If newest is lower than the current min than new is the min. -- XXX Otherwise if exiting one was equal to min only then we need to find -- new min - let rs = min i n * SIZE_OF(a) - rh1 = if i <= n then 0 else rh + let rs = i * SIZE_OF(a) + rh1 = if i < n then 0 else rh in ringRange $ RingArray mba rs rh1 -- | Find the minimum element in a rolling window. diff --git a/test/Streamly/Test/Data/Fold/Combinators.hs b/test/Streamly/Test/Data/Fold/Combinators.hs index 8df101b144..8a663e7165 100644 --- a/test/Streamly/Test/Data/Fold/Combinators.hs +++ b/test/Streamly/Test/Data/Fold/Combinators.hs @@ -469,6 +469,24 @@ partitionBy = do (Stream.fromList [1,2,3,4,5 :: Int]) r `shouldBe` (3, 2) +-- Fold.windowRange (Fold/Window.hs, no separate test module): compare with a +-- list reference on every prefix of an input longer than the window, for +-- window sizes 1 to 4, covering the window filling up and the ring head +-- wrapping around. +windowRange :: Expectation +windowRange = mapM_ checkWindow [1 .. 4] + where + xs = [5, 1, 4, 2, 8, 3, 7, 6, 0, 9] :: [Int] + + range [] = Nothing + range ys = Just (Prelude.minimum ys, Prelude.maximum ys) + + checkPrefix n k = + Stream.fold (Fold.windowRange n) (Stream.fromList (take k xs)) + `shouldReturn` range (drop (max 0 (k - n)) (take k xs)) + + checkWindow n = mapM_ (checkPrefix n) [0 .. length xs] + moduleName :: String moduleName = "Data.Fold.Combinators" @@ -490,6 +508,7 @@ main = hspec $ do prop "findIndex" $ findIndex predicate prop "elemIndex" $ elemIndex 10 prop "null" null + it "windowRange" windowRange prop "elem" $ elem 10 prop "notElem" $ notElem 10 prop "all" $ all predicate diff --git a/test/Streamly/Test/Data/RingArray.hs b/test/Streamly/Test/Data/RingArray.hs index fcecb42c2b..57914495cb 100644 --- a/test/Streamly/Test/Data/RingArray.hs +++ b/test/Streamly/Test/Data/RingArray.hs @@ -351,6 +351,21 @@ testScanRingsOf = do $ Stream.postscanl (RingArray.scanRingsOf 3) (Stream.fromList [1 .. 5 :: Int]) res `shouldBe` [[1], [1, 2], [1, 2, 3], [2, 3, 4], [3, 4, 5]] +-- Compare scanRingsOf with a list reference over an input longer than the +-- ring, for ring sizes 1 to 4, covering the ring filling up and the ring head +-- wrapping around. +testScanRingsOfRef :: IO () +testScanRingsOfRef = mapM_ check [1 .. 4] + where + xs = [5, 1, 4, 2, 8, 3, 7, 6, 0, 9] :: [Int] + + check n = do + res <- Stream.toList + $ Stream.mapM RingArray.toList + $ Stream.postscanl (RingArray.scanRingsOf n) (Stream.fromList xs) + res `shouldBe` + [drop (max 0 (k - n)) (take k xs) | k <- [1 .. length xs]] + testScanFoldRingsBy :: IO () testScanFoldRingsBy = do res <- Stream.toList @@ -475,6 +490,7 @@ main = hspec $ do describe "Stream of rings" $ do it "ringsOf" testRingsOf it "scanRingsOf" testScanRingsOf + it "scanRingsOf: list reference" testScanRingsOfRef it "scanFoldRingsBy" testScanFoldRingsBy it "scanCustomFoldRingsBy" testScanCustomFoldRingsBy it "createOfLast" testCreateOfLast diff --git a/test/Streamly/Test/Data/Scanl/Combinators.hs b/test/Streamly/Test/Data/Scanl/Combinators.hs index 4c4916f294..99e539c3af 100644 --- a/test/Streamly/Test/Data/Scanl/Combinators.hs +++ b/test/Streamly/Test/Data/Scanl/Combinators.hs @@ -243,6 +243,48 @@ partitionS = ([Left 1, Right 2, Left 3, Right 4] :: [Either Int Int]) [[], [1], [2], [1, 3], [2, 4]] +-- The index of the first sampled element is the offset. +sampleFromthenOffsetS :: Expectation +sampleFromthenOffsetS = do + let run offset stride = + Stream.toList + $ Stream.postscanl (F.sampleFromthen offset stride F.toList) + $ Stream.fromList [0 .. 9 :: Int] + run 1 3 `shouldReturn` [[1], [1, 4], [1, 4, 7]] + run 5 3 `shouldReturn` [[5], [5, 8]] + +-- An input that the pipe consumes without yielding must not emit an output. +pipeSkipS :: Expectation +pipeSkipS = + Stream.toList + (Stream.postscanl + (F.pipe (Pipe.filter even) F.sum) (Stream.fromList [1 .. 4 :: Int])) + `shouldReturn` [2, 6] + +-- uniq has a valid result before any input. +uniqInitialS :: Expectation +uniqInitialS = do + check F.uniq "aab" [Nothing, Just 'a', Nothing, Just 'b'] + check (F.compose F.uniq F.toList) "ab" + [[Nothing], [Nothing, Just 'a'], [Nothing, Just 'a', Just 'b']] + +-- When one scan terminates the other one must be finalized. +partitionFinalS :: Expectation +partitionFinalS = do + ref <- newIORef (0 :: Int) + let right = + F.Scanl + (\s a -> return (F.Partial (s + a))) + (return (Fold.Partial 0)) + return + (\s -> modifyIORef' ref (+ 1) >> return s) + left = F.take 1 F.length + r <- Stream.toList + $ Stream.postscanl (F.partition left right) + $ Stream.fromList [Right 1, Left 2 :: Either Int Int] + r `shouldBe` [1, 1] + readIORef ref `shouldReturn` 1 + ------------------------------------------------------------------------------- -- Deprecated combinators (aliases for compose / composeMany) ------------------------------------------------------------------------------- @@ -285,6 +327,7 @@ main = hspec $ -- pipe it "pipe" pipeS it "pipe: filter" pipeFilter + it "pipe: skip" pipeSkipS prop "pipe: filterLaw" $ filterLawScanModifier (F.pipe (Pipe.map id)) it "tee" teeS @@ -300,6 +343,9 @@ main = hspec $ $ filterLawScanModifier (\h -> F.partitionBy Right F.sum h) it "partition" partitionS + it "partition: final" partitionFinalS + it "sampleFromthen: offset" sampleFromthenOffsetS + it "uniq: initial" uniqInitialS it "with" withS it "topBy" topByS it "bottomBy" bottomByS diff --git a/test/Streamly/Test/Data/Scanl/Container.hs b/test/Streamly/Test/Data/Scanl/Container.hs index 14074f5035..65c177e625 100644 --- a/test/Streamly/Test/Data/Scanl/Container.hs +++ b/test/Streamly/Test/Data/Scanl/Container.hs @@ -178,6 +178,35 @@ classifyGenericIOS = (IO (Map.Map String Int), Maybe (String, Int)))) classifyInput classifyExpected +-- The inner scan terminates on the first input of a key, further inputs of +-- that key must be ignored. +classifyDoneOnFirstInput :: [(String, Int)] +classifyDoneOnFirstInput = [("A", 1), ("B", 5), ("A", 2), ("B", 6)] + +classifyDoneOnFirstExpected :: [Maybe (String, Int)] +classifyDoneOnFirstExpected = [Just ("A", 1), Just ("B", 5), Nothing, Nothing] + +classifyDoneOnFirstInner :: F.Scanl IO (String, Int) Int +classifyDoneOnFirstInner = F.lmap snd (F.take 1 F.sum) + +classifyDoneOnFirst :: Expectation +classifyDoneOnFirst = do + let check1 scn = + checkPostscanl scn + classifyDoneOnFirstInput classifyDoneOnFirstExpected + check1 (F.classify fst classifyDoneOnFirstInner) + check1 (F.classifyIO fst classifyDoneOnFirstInner) + check1 + (fmap snd + (F.classifyGeneric fst classifyDoneOnFirstInner + :: F.Scanl IO (String, Int) + (IO (Map.Map String Int), Maybe (String, Int)))) + check1 + (fmap snd + (F.classifyGenericIO fst classifyDoneOnFirstInner + :: F.Scanl IO (String, Int) + (IO (Map.Map String Int), Maybe (String, Int)))) + moduleName :: String moduleName = "Data.Scanl.Container" @@ -207,4 +236,5 @@ main = hspec $ it "classifyGeneric" classifyGenericS it "classifyGeneric: filter" classifyGenericFilterS it "classifyGenericIO" classifyGenericIOS + it "classify: done on first input" classifyDoneOnFirst it "classifyGenericIO: filter" classifyGenericFilterIOS diff --git a/test/Streamly/Test/Data/Scanl/Window.hs b/test/Streamly/Test/Data/Scanl/Window.hs index aa49f18892..b82ca972d5 100644 --- a/test/Streamly/Test/Data/Scanl/Window.hs +++ b/test/Streamly/Test/Data/Scanl/Window.hs @@ -8,6 +8,7 @@ import qualified Streamly.Internal.Data.Scanl as Scanl import qualified Streamly.Internal.Data.Stream as S import Prelude hiding (sum, maximum, minimum) +import qualified Prelude -- Inner scan returns Continue for odd inputs and Partial (window sum) for even. -- Verifies that incrScanWith propagates Continue without emitting output. @@ -36,6 +37,23 @@ incrScanWithFilterS = do return return +-- Compare windowRange with a list reference over inputs longer than the +-- window, for window sizes 1 to 4, covering the window filling up and the ring +-- head wrapping around. +windowRangeRefS :: Expectation +windowRangeRefS = mapM_ check [1 .. 4] + where + xs = [5, 1, 4, 2, 8, 3, 7, 6, 0, 9] :: [Int] + + window n k = drop (max 0 (k - n)) (take k xs) + + range [] = Nothing + range ys = Just (Prelude.minimum ys, Prelude.maximum ys) + + check n = + S.toList (S.scanl (Scanl.windowRange n) (S.fromList xs)) + `shouldReturn` [range (window n k) | k <- [0 .. length xs]] + moduleName :: String moduleName = "Window" @@ -158,6 +176,7 @@ main = hspec $ do [ Nothing, Just (1, 1), Just (1, 2), Just (1, 3), Just (2, 4) , Just (3, 5) ] + it "windowRange: list reference" windowRangeRefS it "windowMinimum" $ scl (Scanl.windowMinimum 3) ([1, 2, 3, 4, 5] :: [Int]) `shouldReturn` diff --git a/test/streamly-tests.cabal b/test/streamly-tests.cabal index ad9a8c9f5c..fc49b930f2 100644 --- a/test/streamly-tests.cabal +++ b/test/streamly-tests.cabal @@ -35,6 +35,12 @@ flag dev manual: True default: False +flag deprecated-tests + description: Include the tests of deprecated modules, requires the + internal-dev flag of streamly + manual: True + default: False + flag has-llvm description: Use llvm backend for better performance manual: True @@ -204,7 +210,7 @@ library Streamly.Test.Data.Parser.CommonTests if !flag(use-streamly-core) exposed-modules: Streamly.Test.Control.Exception.Common - if !flag(use-streamly-core) && flag(dev) + if !flag(use-streamly-core) && flag(deprecated-tests) exposed-modules: Streamly.Test.Prelude.Common ------------------------------------------------------------------------------- @@ -646,7 +652,7 @@ test-suite Prelude ghc-options: -main-is Streamly.Test.Prelude.main if flag(use-streamly-core) buildable: False - if !flag(dev) + if !flag(deprecated-tests) buildable: False test-suite Prelude.Ahead @@ -656,7 +662,7 @@ test-suite Prelude.Ahead ghc-options: -main-is Streamly.Test.Prelude.Ahead.main if flag(use-streamly-core) buildable: False - if !flag(dev) + if !flag(deprecated-tests) buildable: False test-suite Prelude.Async @@ -666,7 +672,7 @@ test-suite Prelude.Async ghc-options: -main-is Streamly.Test.Prelude.Async.main if flag(use-streamly-core) buildable: False - if !flag(dev) + if !flag(deprecated-tests) buildable: False test-suite Prelude.Concurrent @@ -678,7 +684,7 @@ test-suite Prelude.Concurrent ghc-options: +RTS -M2000M -RTS if flag(use-streamly-core) buildable: False - if !flag(dev) + if !flag(deprecated-tests) buildable: False test-suite Prelude.Fold @@ -688,7 +694,7 @@ test-suite Prelude.Fold ghc-options: -main-is Streamly.Test.Prelude.Fold.main if flag(use-streamly-core) buildable: False - if !flag(dev) + if !flag(deprecated-tests) buildable: False test-suite Prelude.Parallel @@ -698,7 +704,7 @@ test-suite Prelude.Parallel ghc-options: -main-is Streamly.Test.Prelude.Parallel.main if flag(use-streamly-core) buildable: False - if !flag(dev) + if !flag(deprecated-tests) buildable: False test-suite Prelude.Rate @@ -706,7 +712,7 @@ test-suite Prelude.Rate type: exitcode-stdio-1.0 main-is: Streamly/Test/Prelude/Rate.hs ghc-options: -main-is Streamly.Test.Prelude.Rate.main - if flag(dev) + if flag(deprecated-tests) buildable: True else buildable: False @@ -722,7 +728,7 @@ test-suite Prelude.Serial ghc-options: +RTS -M1500M -RTS if flag(use-streamly-core) buildable: False - if !flag(dev) + if !flag(deprecated-tests) buildable: False test-suite Prelude.Top @@ -731,7 +737,7 @@ test-suite Prelude.Top main-is: Streamly/Test/Prelude/Top.hs if flag(use-streamly-core) buildable: False - if !flag(dev) + if !flag(deprecated-tests) buildable: False test-suite Prelude.WAsync @@ -741,7 +747,7 @@ test-suite Prelude.WAsync ghc-options: -main-is Streamly.Test.Prelude.WAsync.main if flag(use-streamly-core) buildable: False - if !flag(dev) + if !flag(deprecated-tests) buildable: False test-suite Prelude.WSerial @@ -751,7 +757,7 @@ test-suite Prelude.WSerial ghc-options: -main-is Streamly.Test.Prelude.WSerial.main if flag(use-streamly-core) buildable: False - if !flag(dev) + if !flag(deprecated-tests) buildable: False test-suite Prelude.ZipAsync @@ -763,7 +769,7 @@ test-suite Prelude.ZipAsync ghc-options: +RTS -M750M -RTS if flag(use-streamly-core) buildable: False - if !flag(dev) + if !flag(deprecated-tests) buildable: False test-suite Prelude.ZipSerial @@ -773,5 +779,5 @@ test-suite Prelude.ZipSerial ghc-options: -main-is Streamly.Test.Prelude.ZipSerial.main if flag(use-streamly-core) buildable: False - if !flag(dev) + if !flag(deprecated-tests) buildable: False