[fix] Bignum: use correct site for coerced comparisons

big_op() computed the correct CallSite for >, >=, <, <= in a switch
but then ignored it and always passed sites(context).op_gt to
coerceRelOp. This caused <, <=, and >= to all use > semantics when
comparing a Bignum against a custom type implementing coerce.

For example, (2**100) < MyCoercible.new(2**101) incorrectly returned
false because it used > instead of < after coercion.

Author: kares

core/src/main/java/org/jruby/RubyBignum.java

@@ -919,7 +919,7 @@ private IRubyObject big_op(ThreadContext context, IRubyObject other, BIGNUM_OP_T
         } else if (other instanceof RubyFloat) {
             rel = float_cmp(context, (RubyFloat)other);
         } else {
-            CallSite site;
+            CallSite site = sites(context).op_gt;
             switch (op) {
                 case BIGNUM_OP_GT:
                     site = sites(context).op_gt;
@@ -934,7 +934,7 @@ private IRubyObject big_op(ThreadContext context, IRubyObject other, BIGNUM_OP_T
                     site = sites(context).op_le;
                     break;
             }
-            return coerceRelOp(context, sites(context).op_gt, other);
+            return coerceRelOp(context, site, other);
         }
 
         if (rel.isNil()) return context.fals;

spec/regression/bignum_coerce_relop_spec.rb

@@ -0,0 +1,42 @@
+require 'rspec'
+
+# Regression test for RubyBignum.big_op coercion.
+#
+# When a Bignum is compared against a custom numeric type that
+# implements coerce, the coerced comparison must use the correct
+# operator (>, >=, <, <=). The bug was that all four operators
+# used > after coercion, so < and <= returned wrong results.
+
+describe "Bignum comparison with coerce" do
+  before :all do
+    @cls = Class.new do
+      def initialize(v) @v = v end
+      def coerce(other) [self.class.new(other), self] end
+      def >(other)  @v.to_i > other.to_i  end
+      def >=(other) @v.to_i >= other.to_i end
+      def <(other)  @v.to_i < other.to_i  end
+      def <=(other) @v.to_i <= other.to_i end
+      def to_i()    @v.to_i               end
+    end
+  end
+
+  it "uses > after coercion for >" do
+    expect((2**100) > @cls.new(1)).to be true
+    expect((2**100) > @cls.new(2**101)).to be false
+  end
+
+  it "uses >= after coercion for >=" do
+    expect((2**100) >= @cls.new(1)).to be true
+    expect((2**100) >= @cls.new(2**101)).to be false
+  end
+
+  it "uses < after coercion for <" do
+    expect((2**100) < @cls.new(2**101)).to be true
+    expect((2**100) < @cls.new(1)).to be false
+  end
+
+  it "uses <= after coercion for <=" do
+    expect((2**100) <= @cls.new(2**101)).to be true
+    expect((2**100) <= @cls.new(1)).to be false
+  end
+end