enforce verify_hostname during SSLSocket#connect

CRuby checks verify_hostname inside the OpenSSL verify callback during
the TLS handshake (ossl_ssl.c, depth 0). JSSE has no equivalent hook,
so we check after the handshake completes — connect raises SSLError on
hostname mismatch, matching CRuby behavior. Libraries like net/http and
net/imap that already call post_connection_check will double-check
harmlessly.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: kares

src/main/java/org/jruby/ext/openssl/SSLSocket.java

@@ -321,6 +321,15 @@ private IRubyObject connectImpl(final ThreadContext context, final boolean block
         catch (NotYetConnectedException e) {
             throw newErrnoEPIPEError(context.runtime, "SSL_connect");
         }
+
+        // CRuby enforces verify_hostname inside the OpenSSL verify callback
+        // during the handshake (ossl_ssl.c ossl_ssl_verify_callback, depth 0).
+        // JSSE has no equivalent hook, so we check after the handshake completes.
+        // This is functionally equivalent — connect raises SSLError on mismatch.
+        // Note: net/http and net/imap also call post_connection_check explicitly,
+        // so this may double-check in those cases (harmless, same result).
+        verifyHostnameIfRequired(context);
+
         return this;
     }
 
@@ -410,6 +419,18 @@ private IRubyObject acceptImpl(final ThreadContext context, final boolean blocki
         return this;
     }
 
+    private void verifyHostnameIfRequired(final ThreadContext context) {
+        final IRubyObject verifyHostname = sslContext.getInstanceVariable("@verify_hostname");
+        if (verifyHostname == null || !verifyHostname.isTrue()) return;
+
+        final IRubyObject hostname = getInstanceVariable("@hostname");
+        if (hostname == null || hostname.isNil()) return;
+
+        // delegates to post_connection_check (defined in Ruby) which calls
+        // OpenSSL::SSL.verify_certificate_identity(peer_cert, hostname)
+        callMethod(context, "post_connection_check", hostname);
+    }
+
     final IRubyObject verify_mode(final ThreadContext context) {
         final CallSite[] sites = getMetaClass().getExtraCallSites();
         if (sites == null) return fallback_verify_mode(context);

src/test/ruby/ssl/test_ssl.rb

@@ -119,6 +119,76 @@ def create_cert_with_san(san)
 
   public
 
+  # Ported from CRuby's test_verify_hostname_on_connect (test/openssl/test_ssl.rb).
+  # Verifies that SSLSocket#connect enforces verify_hostname automatically.
+  # On CRuby this is checked inside the OpenSSL verify callback during handshake;
+  # on JRuby it is checked after the JSSE handshake completes (equivalent effect).
+  def test_verify_hostname_on_connect
+    now = Time.now
+    exts = [
+      ["keyUsage", "keyEncipherment,digitalSignature", true],
+      ["subjectAltName", "DNS:a.example.com,DNS:*.b.example.com", false],
+    ]
+    @svr_cert = issue_cert(@svr, @svr_key, 4, exts, @ca_cert, @ca_key,
+                           not_before: now, not_after: now + 1800)
+
+    start_server0(PORT, OpenSSL::SSL::VERIFY_NONE, true) do |server, port|
+      ctx = OpenSSL::SSL::SSLContext.new
+      ctx.verify_hostname = true
+      ctx.cert_store = OpenSSL::X509::Store.new
+      ctx.cert_store.add_cert(@ca_cert)
+      ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
+
+      [
+        ["a.example.com", true],
+        ["A.Example.Com", true],
+        ["x.example.com", false],
+        ["b.example.com", false],
+        ["x.b.example.com", true],
+      ].each do |name, expected_ok|
+        begin
+          sock = TCPSocket.new("127.0.0.1", port)
+          ssl = OpenSSL::SSL::SSLSocket.new(sock, ctx)
+          ssl.hostname = name
+          if expected_ok
+            ssl.connect
+          else
+            assert_raise(OpenSSL::SSL::SSLError) { ssl.connect }
+          end
+        ensure
+          ssl&.close rescue nil
+          sock&.close rescue nil
+        end
+      end
+    end
+  end
+
+  def test_verify_hostname_not_enforced_when_disabled
+    now = Time.now
+    exts = [
+      ["keyUsage", "keyEncipherment,digitalSignature", true],
+      ["subjectAltName", "DNS:a.example.com", false],
+    ]
+    @svr_cert = issue_cert(@svr, @svr_key, 4, exts, @ca_cert, @ca_key,
+                           not_before: now, not_after: now + 1800)
+
+    start_server0(PORT, OpenSSL::SSL::VERIFY_NONE, true) do |server, port|
+      # verify_hostname defaults to false/nil — mismatched hostname should succeed
+      ctx = OpenSSL::SSL::SSLContext.new
+      ctx.cert_store = OpenSSL::X509::Store.new
+      ctx.cert_store.add_cert(@ca_cert)
+      ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
+
+      sock = TCPSocket.new("127.0.0.1", port)
+      ssl = OpenSSL::SSL::SSLSocket.new(sock, ctx)
+      ssl.hostname = "wrong.example.com"
+      ssl.connect # should succeed — verify_hostname is not set
+    ensure
+      ssl&.close rescue nil
+      sock&.close rescue nil
+    end
+  end
+
   def test_post_connect_check_with_anon_ciphers
     unless OpenSSL::ExtConfig::TLS_DH_anon_WITH_AES_256_GCM_SHA384
       return skip('OpenSSL::ExtConfig::TLS_DH_anon_WITH_AES_256_GCM_SHA384 not enabled')