set verify_result to V_ERR_HOSTNAME_MISMATCH on hostname fail

CRuby calls verify_certificate_identity in the verify callback and sets
V_ERR_HOSTNAME_MISMATCH only when it returns `false`.

Mirror that logic post-handshake.

The verify_callback doesn't see this error on JRuby (JSSE limitation,
documented in skipped test).

Author: kares

lib/openssl/ssl.rb

@@ -449,6 +449,14 @@ def post_connection_check(hostname)
         return true
       end
 
+      # @private JRuby due having to call this outside of post_connection_check
+      def verify_certificate_identity_internal(hostname)
+        peer_cert = self.peer_cert
+        return nil if peer_cert.nil?
+        OpenSSL::SSL.verify_certificate_identity(peer_cert, hostname)
+      end
+      private :verify_certificate_identity_internal
+
       # call-seq:
       #   ssl.session -> aSession
       #

src/main/java/org/jruby/ext/openssl/SSLSocket.java

@@ -424,13 +424,19 @@ private void verifyHostnameConnectionCheck(final ThreadContext context) {
         final IRubyObject hostname = getInstanceVariable("@hostname");
         if (hostname == null || hostname.isNil()) return;
 
-        // delegates to post_connection_check (defined in Ruby) which calls
-        // OpenSSL::SSL.verify_certificate_identity(peer_cert, hostname)
-        callMethod(context, "post_connection_check", hostname);
-
-        // stash verified hostname so a subsequent explicit post_connection_check
-        // (e.g. from net/http) does not execute the check twice (to match MRI)
-        setInstanceVariable("@verified_hostname", hostname);
+        // CRuby calls verify_certificate_identity inside the OpenSSL verify
+        // callback and sets V_ERR_HOSTNAME_MISMATCH only when it returns false.
+        IRubyObject result = callMethod(context, "verify_certificate_identity_internal", hostname);
+
+        if (result.isTrue()) {
+            // stash verified hostname so a subsequent explicit post_connection_check
+            // (e.g. from net/http) does not execute the check twice
+            setInstanceVariable("@verified_hostname", hostname);
+        } else if (!result.isNil()) { // false
+            sslContext.setLastVerifyResult(verifyResult = X509Utils.V_ERR_HOSTNAME_MISMATCH);
+            throw newSSLError(context.runtime, // same as `post_connection_check(hostname)`
+                    "hostname \"" + hostname + "\" does not match the server certificate");
+        }
     }
 
     final IRubyObject verify_mode(final ThreadContext context) {

src/test/ruby/ssl/test_ssl.rb

@@ -274,6 +274,79 @@ def test_verify_result_with_verify_peer_self_signed
     end
   end
 
+  # verify_result should report V_ERR_HOSTNAME_MISMATCH when hostname
+  # verification fails during connect (matches CRuby behavior).
+  def test_verify_result_hostname_mismatch
+    now = Time.now
+    exts = [
+      ["keyUsage", "keyEncipherment,digitalSignature", true],
+      ["subjectAltName", "DNS:a.example.com", false],
+    ]
+    @svr_cert = issue_cert(@svr, @svr_key, 4, exts, @ca_cert, @ca_key,
+                           not_before: now, not_after: now + 1800)
+
+    start_server0(PORT, OpenSSL::SSL::VERIFY_NONE, true) do |server, port|
+      ctx = OpenSSL::SSL::SSLContext.new
+      ctx.verify_hostname = true
+      ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      ctx.cert_store = OpenSSL::X509::Store.new
+      ctx.cert_store.add_cert(@ca_cert)
+
+      sock = TCPSocket.new("127.0.0.1", port)
+      ssl = OpenSSL::SSL::SSLSocket.new(sock, ctx)
+      ssl.hostname = "b.example.com"
+      assert_raise(OpenSSL::SSL::SSLError) { ssl.connect }
+      assert_equal OpenSSL::X509::V_ERR_HOSTNAME_MISMATCH, ssl.verify_result
+    ensure
+      ssl&.close rescue nil
+      sock&.close rescue nil
+    end
+  end
+
+  # Ported from CRuby's test_verify_hostname_failure_error_code.
+  # CRuby invokes verify_callback with V_ERR_HOSTNAME_MISMATCH because the
+  # hostname check runs inside OpenSSL's verify callback during handshake.
+  # JRuby checks hostname post-handshake (JSSE limitation), so the callback
+  # doesn't see the error. Skipped on JRuby; runs on CRuby for parity check.
+  def test_verify_hostname_failure_error_code_via_callback
+    skip 'verify_callback not invoked for hostname mismatch (JSSE limitation)' if defined?(JRUBY_VERSION)
+
+    now = Time.now
+    exts = [
+      ["keyUsage", "keyEncipherment,digitalSignature", true],
+      ["subjectAltName", "DNS:a.example.com", false],
+    ]
+    @svr_cert = issue_cert(@svr, @svr_key, 4, exts, @ca_cert, @ca_key,
+                           not_before: now, not_after: now + 1800)
+
+    start_server0(PORT, OpenSSL::SSL::VERIFY_NONE, true) do |server, port|
+      verify_callback_ok = verify_callback_err = nil
+
+      ctx = OpenSSL::SSL::SSLContext.new
+      ctx.verify_hostname = true
+      ctx.cert_store = OpenSSL::X509::Store.new
+      ctx.cert_store.add_cert(@ca_cert)
+      ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      ctx.verify_callback = -> (preverify_ok, store_ctx) {
+        verify_callback_ok = preverify_ok
+        verify_callback_err = store_ctx.error
+        preverify_ok
+      }
+
+      begin
+        sock = TCPSocket.new("127.0.0.1", port)
+        ssl = OpenSSL::SSL::SSLSocket.new(sock, ctx)
+        ssl.hostname = "b.example.com"
+        assert_raise(OpenSSL::SSL::SSLError) { ssl.connect }
+        assert_equal false, verify_callback_ok
+        assert_equal OpenSSL::X509::V_ERR_HOSTNAME_MISMATCH, verify_callback_err
+      ensure
+        ssl&.close rescue nil
+        sock&.close rescue nil
+      end
+    end
+  end
+
   def test_post_connect_check_with_anon_ciphers
     unless OpenSSL::ExtConfig::TLS_DH_anon_WITH_AES_256_GCM_SHA384
       return skip('OpenSSL::ExtConfig::TLS_DH_anon_WITH_AES_256_GCM_SHA384 not enabled')