Refactor scip-ruby repo tests and add packaging coverage

.buildkite/test-indexer.sh

@@ -79,21 +79,11 @@ test_args=(
   --test_output=errors \
   "${test_args[@]}" || err=$?
 
-echo "--- Installing Ruby"
-
-if (! file /root/.asdf/installs/ruby/2.7.0) || (! asdf global ruby 2.7.0); then
-  rm -rf .asdf/shims
-  OPENSSL_CFLAGS=-Wno-error=implicit-function-declaration asdf install ruby
-  asdf global ruby 2.7.0
-fi
-
 echo "+++ Running repo tests"
 
 test_args=(
   "//test/scip/repos"
   "-c" "opt"
-  "--test_env" "PATH=${PATH}"
-  "--test_env" "HOME=${HOME}"
   "--config=forcedebug"
   "--spawn_strategy=local"
 )
@@ -106,6 +96,23 @@ test_args=(
   --test_output=errors \
   "${test_args[@]}" || err=$?
 
+echo "+++ Running packaging tests"
+
+test_args=(
+  "//test/scip/packaging"
+  "-c" "opt"
+  "--config=forcedebug"
+  "--spawn_strategy=local"
+)
+
+./bazel test \
+  --experimental_generate_json_trace_profile \
+  --profile=_out_/profile_packaging_tests.json \
+  --experimental_execution_log_file=_out_/packaging_test.log \
+  --test_summary=terse \
+  --test_output=errors \
+  "${test_args[@]}" || err=$?
+
 if [ "$err" -ne 0 ]; then
   echo "--- annotating build result"
   failing_tests="$(mktemp)"

.github/workflows/ci.yml

@@ -43,35 +43,7 @@ jobs:
           path: log
       - name: "🏋️‍♂️ Run tests"
         run: ./bazel test //test/scip --config=dbg
-      # Repo tests are kinda' broken right now because the bundle cache step needs synchronization
-      #
-      # - name: "🏋️‍♂️ Build repo tests"
-      #   continue-on-error: true
-      #   run: |
-      #     sudo apt-get install --reinstall build-essential
-      #     ls "$(dirname "$(command -v gcc)")"
-      #     echo "$PATH"
-      #     ls /usr/bin /usr/local/bin /bin
-      #     # Technically, this is for Ruby 2.7.0 but it's probably fine.
-      #     sudo apt-get install ruby2.7-dev
-      #     ./bazel test //test/scip/repos --test_env GITHUB_ACTIONS=1 --test_env GEM_PATH="$(dirname "$(command -v gem)")" --config=dbg
-      #
-      # Blocked on a bundler bug in 2.3.4. I get a crash trace like this when running locally.
-      #
-      # NoMethodError: undefined method `metadata' for nil:NilClass
-      #   /Users/varun/Code/scip-ruby/.cache_ruby/versions/2.7.2/lib/ruby/gems/2.7.0/gems/bundler-2.3.4/lib/bundler/cli/common.rb:22:in `block in output_fund_metadata_summary'
-      #   /Users/varun/Code/scip-ruby/.cache_ruby/versions/2.7.2/lib/ruby/gems/2.7.0/gems/bundler-2.3.4/lib/bundler/cli/common.rb:22:in `count'
-      #   /Users/varun/Code/scip-ruby/.cache_ruby/versions/2.7.2/lib/ruby/gems/2.7.0/gems/bundler-2.3.4/lib/bundler/cli/common.rb:22:in `output_fund_metadata_summary'
-      #   /Users/varun/Code/scip-ruby/.cache_ruby/versions/2.7.2/lib/ruby/gems/2.7.0/gems/bundler-2.3.4/lib/bundler/cli/install.rb:87:in `run'
-      #
-      # This is the method in question:
-      #
-      # def self.output_fund_metadata_summary
-      #   definition = Bundler.definition
-      #   current_dependencies = definition.requested_dependencies
-      #   current_specs = definition.specs
-      #
-      #   count = current_dependencies.count {|dep| current_specs[dep.name].first.metadata.key?("funding_uri") }
-      #                                                                          ^ boom!
-      # We should probably file a bug against bundler but I haven't been able to repro
-      # the issue outside of a Bazel sandbox yet.
+      - name: "🏋️‍♂️ Run repo tests"
+        run: ./bazel test //test/scip/repos --config=dbg
+      - name: "🏋️‍♂️ Run packaging tests"
+        run: ./bazel test //test/scip/packaging --config=dbg

docs/scip-ruby/CONTRIBUTING.md

@@ -44,8 +44,8 @@ see the [Design Decisions doc][].
 
 ## Configuring Ruby (optional)
 
-If you're going to be running the repository tests or building the
-scip-ruby gem locally, follow these steps before building stuff.
+If you're going to be building the scip-ruby gem locally, follow these steps
+before building stuff.
 
 ### macOS pre-requisites
 
@@ -180,19 +180,15 @@ Snapshot outputs can be easily updated:
 ./bazel test --config=dev //test/scip:update_alias
 ```
 
-Repo tests are kinda' broken right now; they're disabled
-in CI (see ci.yml), and may or may not work on your machine.
-
-If you want to run repo tests, first complete the
-[Configuring Ruby](#configuring-ruby-optional) steps.
-Then run the tests using:
+Repo tests index pinned OSS projects using the `scip-ruby` binary built from
+this checkout. Run them using:
 
 ```bash
-# If Ruby was installed via asdf (recommended to avoid dependency on system Ruby on macOS)
 ./bazel test //test/scip/repos --config=dev
 ```
 
-This may take a few minutes to run.
+These tests currently check that indexing succeeds and creates a non-empty
+`index.scip` file.
 
 ### Writing a new snapshot test
 
@@ -210,26 +206,20 @@ and follow the same structure.
 
 ### Writing a new repo test
 
-First, clone the repo using Sorbet locally
-and check if you can index it.
-Typically, the commands will be something like:
+First, add the project archive information to
+[`third_party/test_gem_data.bzl`](../../third_party/test_gem_data.bzl).
+The archive should be pinned to a specific version and checksum.
 
-```bash
-BUNDLE_WITH=sorbet bundle install
+Then check that the project can be indexed with the built binary. Typically,
+the command will look like:
 
-# Replace srb binary with scip-ruby binary
-cp /path/to/scip-ruby "$(find . -name 'srb' -type f | head -n 1)"
-bundle exec srb --index-file index.scip --gem-metadata "name@version"
-```
-
-In case there are any type errors, create a patch and save it:
 ```bash
-git diff > /path/to/test/scip/repos/name-version.patch
+./bazel build //main:scip-ruby --config=dev
+./bazel-bin/main/scip-ruby --index-file index.scip --gem-metadata "name@version"
 ```
 
-Once you're able to successfully index the code,
-modify the [scip_repos_test.bzl](test/scip/repos/scip_repos_test.bzl)
-file to include the relevant data.
+Once you're able to successfully index the code, run
+`./bazel test //test/scip/repos --config=dev`.
 
 ## Debugging
 

docs/scip-ruby/DESIGN.md

@@ -1,81 +1,56 @@
 # Design Decisions
 
-## Repo tests
-
-<!-- DEF NOTE[repo-test-structure] -->
-
-WARNING: Right now, the repo tests are broken/disabled,
-so this information is a bit inaccurate. We should fix
-this once the repo tests are fixed.
+## Repo tests and packaging tests have separate jobs
 
-The repo tests have a couple of unusual things:
-#. Tests are broken up into 3 stages,
-   instead of being a simple script that does clone
-   + apply patch + invoke dev tool (`scip-ruby`).
-#. There is "manual" caching going on for the intermediate results
-   of these steps. The cache cannot be used in parallel; repo tests run serially.
+Repo tests and packaging tests intentionally cover different risks.
 
-I'll cover each of these one-by-one.
+Repo tests answer: "Can the indexer built from this checkout process real Ruby
+projects?" They use the built `scip-ruby` binary directly against pinned OSS
+projects.
 
-Some constraints to keep in mind while reading the sections below:
-#. We want to run tests using as close to "real-world" tools (such as Bundler)
-   as possible instead of trying to mimic what they do.
-#. We want to cache intermediate results as much as possible,
-   as many steps in Ruby installation + Gem installation are slow.
-#. Tests should hopefully run reasonably quickly, especially if you only
-   modify the indexer (most common situation).
-#. Tests should be isolated from each other. We don't one test to "install"
-   `scip-ruby` and have that be visible to another test.
+Packaging tests answer: "Can the packaged gem be installed and run the way a
+user would get it?" They install the built gem into a small temporary Ruby
+project and run `scip-ruby` from that installation.
 
-Before going into details about the caching, first let's cover
-an outline of the 3 stages.
-### 3 stages of repo tests
+Keeping these tests separate makes each failure easier to understand. A repo
+test failure points at indexing behavior on a real project. A packaging test
+failure points at gem building, gem installation, Bundler setup, or the packaged
+launcher.
 
-#. Create a standalone ruby installation: This is cached
-   and the installation is blown away.
-#. Test prep: This step involves installing the dependencies of the gem,
-   including any custom bundler version it may require.
-   The pristine toolchain from the first step is extracted
-   and we run `bundle cache`.
-   After this is done, the _entire_ Ruby toolchain is cached again
-   (because it will be have been modified by `bundle cache`).
-   The modified source tree is also cached (I tried `--frozen` earlier but
-   ran into some error with it that I didn't have time to look into).
-#. Index: This step reuses the two caches from the previous step,
-   applies a patch, copies the `scip-ruby` gem into the `vendor/cache`
-   directory, runs `bundle install --local` (so nothing should be
-   fetched from the internet). Now, everything is in a state which mostly
-   mimics what a user would have. Then we invoke `scip-ruby` normally.
-
-Intermediate results use tarballs for caching directory trees instead of copying
-directories because it seemed much simpler to deal with a tarball in Bazel.
+## Repo tests
 
-Why do we go through all this trouble to cache things "manually"?
-### Manual caching
+<!-- DEF NOTE[repo-test-structure] -->
 
-Here are some obstacles the caching system needs to overcome.
+Repo tests exercise `scip-ruby` against pinned, real-world OSS projects.
+They intentionally do not install the `scip-ruby` gem into those projects.
+Instead, each test:
 
-* Ruby installation is [path-dependent](https://github.com/ruby/setup-ruby#using-self-hosted-runners).
-* Ruby installation is relatively slow as it builds everything from source:
-  it takes a few minutes even on a Macbook Pro.
-  Installing it per-test would be too time-consuming.
+1. unpacks a pinned source archive into a fresh temporary directory,
+2. runs the `scip-ruby` binary built from this checkout,
+3. passes explicit gem metadata for the project, and
+4. checks that a non-empty `index.scip` file was created.
 
-These two factors together imply that we should cache a Ruby installation
-inside a stable directory, not a temporary test directory.
+This keeps the tests focused on whether the current indexer can process a
+real project. It avoids modifying third-party lockfiles, installing a freshly
+built gem through Bundler, or mutating a shared Ruby installation while the test
+runs.
 
-Additionally, we only want one Ruby toolchain cache; to ensure test isolation,
-we run them in serial on the same directory (a GIL for the test suite).
+The current check is intentionally lightweight. 
 
-* We want to ensure reproducibility, and not mess with the global environment.
+## Packaging tests
 
-To achieve this, we delete the toolchain directory at the start of a test,
-and restore it from the cache in the 'Test prep' step.
+Packaging tests exercise the built `scip-ruby` gem, not just the built binary.
+They intentionally use a small fixture project instead of the pinned OSS
+projects used by repo tests. Each test:
 
-### Bonus: Bundler weirdness
+1. builds a gem for the current platform,
+2. creates a temporary Ruby project,
+3. installs that gem through Bundler from a local cache,
+4. runs the installed `scip-ruby` command against a sample Ruby file, and
+5. checks that a non-empty `index.scip` file was created.
 
-If you run `bundle install --local` for a test gem X,
-after copying an updated `scip-ruby` gem (same version, new checksum)
-into X's `vendor/cache` directory,
-you would naively expect `bundle` to re-install the updated `scip-ruby` gem.
-(install = copy/setup stuff in toolchain-adjacent directories)
-But `bundle` doesn't actually do that. 🙈
+This keeps packaging coverage focused on the path users rely on when installing
+the gem: the gem file, its platform tag, its executable, its bundled binary, and
+the Ruby/Bundler environment around it. It avoids making every repo test also
+prove gem installation, which would make failures harder to diagnose and would
+slow down the real-project coverage.

gems/scip-ruby/BUILD

@@ -1,4 +1,4 @@
-load(":build_gems.bzl", "build_gems", "setup_standalone_ruby", "version")
+load(":build_gems.bzl", "build_current_platform_gem", "build_gems", "setup_standalone_ruby", "version")
 
 version(
     name = "version",
@@ -26,5 +26,26 @@ build_gems(
         "@platforms//os:linux": "linux",
         "@platforms//os:osx": "darwin",
     }),
+    llvm_libunwind = select({
+        "@platforms//os:osx": ["@llvm_toolchain_15_0_6_llvm//:lib/libunwind.1.0.dylib"],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:public"],
+)
+
+build_current_platform_gem(
+    name = "scip-ruby-current-platform",
+    srcs = [
+        "bin/scip-ruby",
+        "scip-ruby.template.gemspec",
+    ],
+    gem_name = select({
+        "//tools/config:dbg": "scip-ruby-debug",
+        "//conditions:default": "scip-ruby",
+    }),
+    llvm_libunwind = select({
+        "@platforms//os:osx": ["@llvm_toolchain_15_0_6_llvm//:lib/libunwind.1.0.dylib"],
+        "//conditions:default": [],
+    }),
     visibility = ["//visibility:public"],
 )