Scanl fixes

.github/workflows/haskell.yml

@@ -194,7 +194,7 @@ jobs:
 
           # This includes the debug flag as well
           - name: dev-no-opt
-            runner: macos-latest
+            runner: ubuntu-latest
             command: cabal
             ghc_version: 9.14.1
             # WARNING! cannot use # comments inside pack_options.
@@ -246,9 +246,11 @@ jobs:
             runner: ubuntu-latest
             command: cabal
             ghc_version: 9.8.4
+            # The deprecated tests need the deprecated modules of streamly.
             # WARNING! cannot use # comments inside pack_options.
             pack_options: >-
               CABAL_PROJECT=cabal.project
+              CABAL_BUILD_OPTIONS="--flag deprecated-tests --constraint=streamly+internal-dev"
             ignore_error: false
 
           - name: streamly-sdist
@@ -413,7 +415,7 @@ jobs:
       with:
         path: |
             ~/.cache/cabal/packages
-        key: cabal-hackage-index-non-windows
+        key: cabal-hackage-index
 
     # GHC is not test/options specific
     - name: Restore ghcup and ghc (non-Windows)
@@ -554,7 +556,7 @@ jobs:
       if: always() && steps.deps-only.outcome == 'success' && runner.os != 'Windows' && steps.restore-hackage.outputs.cache-hit != 'true'
       with:
         path: ~/.cache/cabal/packages
-        key: cabal-hackage-index-non-windows
+        key: cabal-hackage-index
 
     - name: Save ghcup and ghc (non-Windows)
       uses: actions/cache/save@v4

.github/workflows/js.yml

@@ -97,14 +97,25 @@ jobs:
         cabal build --project-file=cabal.project.ghcjs \
           --builddir=$BUILDDIR all
 
+    # Run the test executables built above instead of using "cabal test".
+    # GHC recompiles the modules that use Template Haskell on every cabal
+    # invocation, "cabal test" would spend about 5 minutes rebuilding them.
+    # Like "cabal test", run each test in the package directory.
     - name: Test
       run: |
-        set +e
-        cabal test --keep-going --project-file=cabal.project.ghcjs \
-          --builddir=$BUILDDIR all 2>&1 | tee test.log
-        set -e
-        grep -E "^Test suite .*: (PASS|FAIL)$" test.log
-        failed=$(sed -n 's/^Test suite \(.*\): FAIL$/\1/p' test.log)
+        tests=$(jq -r '."install-plan"[]
+          | select(."pkg-name" == "streamly-tests"
+                   and ((."component-name" // "") | startswith("test:")))
+          | ."component-name" + " " + ."bin-file"' $BUILDDIR/cache/plan.json)
+        failed=
+        cd test
+        while read -r comp bin; do
+          t=${comp#test:}
+          echo "::group::$t"
+          if "$bin" < /dev/null; then r=PASS; else r=FAIL; failed="$failed $t"; fi
+          echo "::endgroup::"
+          echo "Test suite $t: $r"
+        done <<< "$tests"
         unexpected=
         for t in $failed; do
           case " $KNOWN_FAILURES " in
@@ -116,8 +127,3 @@ jobs:
           echo "Unexpected failures:$unexpected"
           exit 1
         fi
-        # A test suite that fails to build is not reported as FAIL
-        if grep -q "^Error: \[Cabal-" test.log; then
-          echo "cabal test failed to build some test suites"
-          exit 1
-        fi

.github/workflows/markdown-doctest.yml

@@ -20,22 +20,21 @@ jobs:
       run: |
         rm -f ~/.ghcup
 
-    - name: Cache ghcup and ghc (non-Windows)
-      uses: actions/cache@v4
-      if: runner.os != 'Windows'
+    # Saved by the TEST workflow on master
+    - name: Restore ghcup and ghc
+      uses: actions/cache/restore@v4
       with:
         path: |
           ~/.ghcup
-        key: ${{ runner.os }}-ghcup-${{ env.GHCUP_VERSION }}-${{ env.GHCVER }}
+        key: ghc-${{ env.GHCVER }}-v-ubuntu-latest
 
-    - name: Cache hackage package index (non-Windows)
-      uses: actions/cache@v4
-      if: runner.os != 'Windows'
+    # Saved by the TEST workflow on master, "cabal update" updates it.
+    - name: Restore hackage package index
+      uses: actions/cache/restore@v4
       with:
         path: |
             ~/.cache/cabal/packages
-        # Bump the key version to clear the cache
-        key: cache-cabal-packages
+        key: cabal-hackage-index
 
     # NOTE: we need the cabal state as well because the executable may depend
     # on shared libraries in the cabal store.
@@ -59,7 +58,7 @@ jobs:
 
     - name: Download ghc
       run: |
-        if ! ghc --version 2>/dev/null | grep -q "$GHCVER"
+        if ! ghc --version 2>/dev/null | grep -q "$GHCVER" || ! command -v cabal
         then
           curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VERSION/x86_64-linux-ghcup-$GHCUP_VERSION
           chmod +x ./ghcup

.github/workflows/packdiff.yml

@@ -20,22 +20,21 @@ jobs:
       run: |
         rm -f ~/.ghcup
 
-    - name: Cache ghcup and ghc (non-Windows)
-      uses: actions/cache@v4
-      if: runner.os != 'Windows'
+    # Saved by the TEST workflow on master
+    - name: Restore ghcup and ghc
+      uses: actions/cache/restore@v4
       with:
         path: |
           ~/.ghcup
-        key: ${{ runner.os }}-ghcup-${{ env.GHCUP_VERSION }}-${{ env.GHCVER }}
+        key: ghc-${{ env.GHCVER }}-v-ubuntu-latest
 
-    - name: Cache hackage package index (non-Windows)
-      uses: actions/cache@v4
-      if: runner.os != 'Windows'
+    # Saved by the TEST workflow on master, "cabal update" updates it.
+    - name: Restore hackage package index
+      uses: actions/cache/restore@v4
       with:
         path: |
             ~/.cache/cabal/packages
-        # Bump the key version to clear the cache
-        key: cache-cabal-packages
+        key: cabal-hackage-index
 
     - name: Cache packdiff (non-Windows)
       uses: actions/cache@v4
@@ -59,7 +58,7 @@ jobs:
 
     - name: Download ghc
       run: |
-        if ! ghc --version 2>/dev/null | grep -q "$GHCVER"
+        if ! ghc --version 2>/dev/null | grep -q "$GHCVER" || ! command -v cabal
         then
           curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VERSION/x86_64-linux-ghcup-$GHCUP_VERSION
           chmod +x ./ghcup

.github/workflows/regression-check.yml

@@ -3,6 +3,11 @@ name: PERF
 on:
   workflow_dispatch:
   pull_request:
+  # On master only the benchmarks are built and cached, so that pull requests
+  # can reuse them.
+  push:
+    branches:
+      - master
 
 # References:
 # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
@@ -70,31 +75,54 @@ jobs:
       run: |
         rm -f ~/.ghcup
 
-    - name: Cache ghcup and ghc (non-Windows)
-      uses: actions/cache@v4
-      if: runner.os != 'Windows'
+    # Saved by the TEST workflow on master
+    - name: Restore ghcup and ghc
+      uses: actions/cache/restore@v4
       with:
         path: |
           ~/.ghcup
-        key: ${{ runner.os }}-ghcup-${{ env.GHCUP_VERSION }}-${{ env.GHCVER }}
+        key: ghc-${{ env.GHCVER }}-v-ubuntu-latest
 
-    - name: Cache hackage package index (non-Windows)
-      uses: actions/cache@v4
-      if: runner.os != 'Windows'
+    # Saved by the TEST workflow on master, "cabal update" updates it.
+    - name: Restore hackage package index
+      uses: actions/cache/restore@v4
       with:
         path: |
             ~/.cache/cabal/packages
-        # Bump the key version to clear the cache
-        key: cache-cabal-packages
+        key: cabal-hackage-index
 
-    - name: Cache cabal build dependencies (non-Windows)
-      uses: actions/cache@v4
-      if: runner.os != 'Windows'
+    - name: Determine the "master" commit
+      run: |
+        if test "${{ github.event_name }}" = push
+        then
+          sha=${{ github.sha }}
+        else
+          sha=$(git ls-remote origin refs/heads/master | cut -f1)
+        fi
+        echo "MASTER_SHA=$sha" >> $GITHUB_ENV
+
+    # Saved by the push to master run of this workflow. A pull request can
+    # restore only the caches saved by itself or by master. When master has
+    # not changed the master benchmarks are up to date and only need to be
+    # run, otherwise an older master build is restored and the build is
+    # incremental. The cabal store is in the same cache so that the
+    # dependencies match the build directory.
+    - name: Restore the "master" build
+      id: master-build
+      uses: actions/cache/restore@v4
       with:
         path: |
-            ~/.local/state/cabal
-        # Bump the key version to clear the cache
-        key: performance-comparison-deps
+          ~/.local/state/cabal
+          dist-newstyle
+        key: perf-master-build-${{ env.GHCVER }}-${{ env.MASTER_SHA }}
+        restore-keys: perf-master-build-${{ env.GHCVER }}-
+
+    - name: Cache bench-runner
+      id: bench-runner
+      uses: actions/cache@v4
+      with:
+        path: bench-runner
+        key: bench-runner-${{ env.GHCVER }}-${{ hashFiles('cabal.project.report', 'benchmark/bench-runner/**', 'targets/**') }}
 
     - name: Update CI_BENCHMARKS environment variable
       run: |
@@ -112,7 +140,7 @@ jobs:
 
     - name: Download ghc
       run: |
-        if ! ghc --version 2>/dev/null | grep -q "$GHCVER"
+        if ! ghc --version 2>/dev/null | grep -q "$GHCVER" || ! command -v cabal
         then
           curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VERSION/x86_64-linux-ghcup-$GHCUP_VERSION
           chmod +x ./ghcup
@@ -124,9 +152,11 @@ jobs:
         # XXX update only if not present
         cabal update
 
+    # Use a separate build directory to keep dist-newstyle for the benchmarks
     - name: Install bench-runner from PR branch
+      if: steps.bench-runner.outputs.cache-hit != 'true'
       run: |
-        cabal install bench-runner --project-file=cabal.project.report --installdir=./ --allow-newer
+        cabal install bench-runner --project-file=cabal.project.report --builddir=dist-bench-runner --installdir=./ --allow-newer
 
     # -----------------------------------------------------------------
     # -- Generate reports for the base branch and upload
@@ -135,10 +165,33 @@ jobs:
     - name: Checkout the "master" branch
       uses: actions/checkout@v4
       with:
-        ref: master
+        ref: ${{ env.MASTER_SHA }}
         clean: false
 
+    # Only build on master. The build command must be the same as the one
+    # that bench-runner uses, otherwise the pull request runs rebuild.
+    - name: Build benchmarks for "master" branch
+      if: github.event_name == 'push'
+      run: |
+        targets=
+        for b in $CI_BENCHMARKS
+        do
+          targets="$targets streamly-benchmarks:bench:$b"
+        done
+        cabal v2-build --flag fusion-plugin --flag limit-build-mem \
+          --with-compiler ghc --enable-benchmarks $targets
+
+    - name: Save the "master" build
+      if: github.event_name == 'push' && steps.master-build.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: |
+          ~/.local/state/cabal
+          dist-newstyle
+        key: ${{ steps.master-build.outputs.cache-primary-key }}
+
     - name: Run benchmarks for "master" branch
+      if: github.event_name != 'push'
       run: |
         # --cabal-build-options "-j1"
         ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw
@@ -151,12 +204,14 @@ jobs:
     # -----------------------------------------------------------------
 
     - name: Checkout the PR branch
+      if: github.event_name != 'push'
       uses: actions/checkout@v4
       with:
         # preserve bench-runner and the "charts" directory created by bench-runner
         clean: false
 
     - name: Run benchmarks and append
+      if: github.event_name != 'push'
       run: |
         # --cabal-build-options "-j1"
         ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw --append
@@ -166,10 +221,12 @@ jobs:
     # -----------------------------------------------------------------
 
     - name: List all benchmarks
+      if: github.event_name != 'push'
       run: |
         ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --no-measure
 
     - name: Compare benchmarks
+      if: github.event_name != 'push'
       run: |
 
         EXIT_STATUS=0

core/docs/Changelog.md

@@ -23,9 +23,15 @@
   separator, e.g. separator `"abc"` and input `"abcab"`.
 * Fix `chunksOf` in the array modules emitting a trailing empty array when
   the stream length is a multiple of the chunk size.
-* Fix `Fold.classifyScan`, `classifyScanIO`, `classify` and `classifyIO`
-  restarting the fold for a key that terminated on its first input.
+* Fix `classify` and `classifyIO` in Fold and Scanl, and
+  `Fold.classifyScan` and `classifyScanIO`, restarting the fold for a key
+  that terminated on its first input.
 * Fix the `Arrow` instance of `Scan` failing when a scan skips an input.
+* Fix `RingArray.ringsOf` and `scanRingsOf` producing rings of a wrong size
+  after `2^31` bytes when `Int` is 32-bit, e.g. on the JavaScript backend.
+* Fix `Scanl.uniq` and `uniqBy` failing before the first input.
+* Fix `Scanl.partition`, `partitionBy` and `partitionByM` not finalizing one
+  scan when the other terminates.
 
 * On POSIX platforms, the `Path` based APIs that open an existing file in
   `WriteMode`, e.g. `write` and `withFile` in `Streamly.FileSystem.FileIO`,

core/src/Streamly/Internal/Data/Fold/Window.hs

@@ -301,7 +301,9 @@ windowRange n = Fold step initial extract extract
 
     step (Tuple3Fused' mba rh i) a = do
         RingArray _ _ rh1 <- RingArray.replace_ (RingArray mba (n * SIZE_OF(a)) rh) a
-        return $ Partial $ Tuple3Fused' mba rh1 (i + 1)
+        -- i is the number of elements in the window, at most n
+        let i1 = min (i + 1) n
+        return $ Partial $ Tuple3Fused' mba rh1 i1
 
     -- XXX exitify optimization causes a problem here when modular folds are
     -- used. Sometimes inlining "extract" is helpful.
@@ -311,8 +313,8 @@ windowRange n = Fold step initial extract extract
     -- XXX If exiting one was equal to min only then we need to find new min
     -- XXX We can supply a custom extract function to a generic window
     -- operation.
-        let rs = min i n * SIZE_OF(a)
-            rh1 = if i <= n then 0 else rh
+        let rs = i * SIZE_OF(a)
+            rh1 = if i < n then 0 else rh
          in ringRange $ RingArray mba rs rh1
 
 -- | Find the minimum element in a rolling window.

core/src/Streamly/Internal/Data/RingArray.hs

@@ -539,16 +539,25 @@ scanRingsOf n = Scanl step initial extract extract
 
     step (Tuple3Fused' mba rh offset) a = do
         RingArray _ _ rh1 <- replace_ (RingArray mba rSize rh) a
-        let offset1 = offset + SIZE_OF(a)
+        -- offset is the size of the ring in bytes, at most rSize
+        --
+        -- XXX GHC 9.14.1 does not inline this step in the fold loop, and
+        -- boxes the fold state on every input, if we write this as:
+        --
+        -- > if offset < rSize then offset + SIZE_OF(a) else offset
+        --
+        -- Array.createOfLast 1 then allocates 8 MB for 100,000 inputs
+        -- instead of none. GHC 9.10.3 does not have this problem. Report it
+        -- to GHC.
+        let offset1 = min (offset + SIZE_OF(a)) rSize
         return $ Scanl.Partial $ Tuple3Fused' mba rh1 offset1
 
     -- XXX exitify optimization causes a problem here when modular folds are
     -- used. Sometimes inlining "extract" is helpful.
     {-# INLINE extract #-}
     extract (Tuple3Fused' mba rh offset) =
-        let rs = min offset rSize
-            rh1 = if offset <= rSize then 0 else rh
-         in pure $ RingArray mba rs rh1
+        let rh1 = if offset < rSize then 0 else rh
+         in pure $ RingArray mba offset rh1
 
 -- | @ringsOf n stream@ groups the input stream into a stream of ring arrays of
 -- size up to n. See 'scanRingsOf' for more details.