[fix] invalid Cert#sign handling -> raise

instead of simply causing a ClassCastException

Author: kares

src/main/java/org/jruby/ext/openssl/X509Cert.java

@@ -564,16 +564,23 @@ else if ( "DSA".equalsIgnoreCase(algorithm) ) {
     public IRubyObject sign(final ThreadContext context, final IRubyObject key, final IRubyObject digest) {
         final Ruby runtime = context.runtime;
 
+        if (!(key instanceof PKey)) { // MRI: NoMethodError: undefined method `private?' for nil:NilClass
+            throw runtime.newTypeError(key, PKey._PKey(runtime).getClass("PKey"));
+        }
+
         // Have to obey some artificial constraints of the OpenSSL implementation. Stupid.
         final String keyAlg = ((PKey) key).getAlgorithm();
         final String digAlg; final String digName;
         if (digest instanceof Digest) {
             digAlg = ((Digest) digest).getShortAlgorithm();
-            digName = ((Digest) digest).name().toString();
+            digName = ((Digest) digest).getName();
         }
-        else {
+        else if (digest instanceof RubyString) {
             digAlg = digest.asJavaString(); digName = null;
         }
+        else { // MRI: TypeError: wrong argument type nil (expected OpenSSL/Digest)
+            throw runtime.newTypeError(digest, Digest._Digest(runtime));
+        }
 
         if( ( "DSA".equalsIgnoreCase(keyAlg) && "MD5".equalsIgnoreCase(digAlg) ) ||
             ( "RSA".equalsIgnoreCase(keyAlg) && "DSS1".equals(digName) ) ) {

src/test/ruby/x509/test_x509cert.rb

@@ -361,6 +361,38 @@ def test_to_text_npe_regression
     assert_equal OpenSSL::BN.new(1), cert.serial
   end
 
+  def test_sign_invalid_arg
+    issuer = subject = OpenSSL::X509::Name.new
+    subject.add_entry('C', 'JP')
+    subject.add_entry('ST', 'Tokyo')
+    subject.add_entry('L', 'Chiyoda')
+    subject.add_entry('CN', 'demo.example.com')
+
+    cert = OpenSSL::X509::Certificate.new
+    cert.not_before = Time.at(0)
+    cert.not_after = Time.now + 1 * 365 * 86400
+    cert.public_key = pkey = OpenSSL::PKey::RSA.generate(1024)
+    cert.serial = 1
+    cert.issuer = issuer
+    cert.subject = subject
+    cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)]))
+
+    digest = OpenSSL::Digest::SHA1.new
+    begin
+      cert.sign(nil, digest)
+      fail 'expected sign to fail (on pkey)'
+    rescue StandardError # expected
+      assert :ok
+    end
+
+    begin
+      cert.sign(pkey, nil)
+      fail 'expected sign to fail (on digest)'
+    rescue TypeError # expected
+      assert :ok
+    end
+  end
+
   def test_cert_loading_regression
     cert_text = "0\x82\x01\xAD0\x82\x01\xA1\xA0\x03\x02\x01\x02\x02\x01\x010\x03\x06\x01\x000g1\v0\t\x06\x03U\x04\x06\x13\x02US1\x130\x11\x06\x03U\x04\b\f\nCalifornia1\x150\x13\x06\x03U\x04\a\f\fSanta Monica1\x110\x0F\x06\x03U\x04\n\f\bOneLogin1\x190\x17\x06\x03U\x04\x03\f\x10app.onelogin.com0\x1E\x17\r100309095845Z\x17\r150309095845Z0g1\v0\t\x06\x03U\x04\x06\x13\x02US1\x130\x11\x06\x03U\x04\b\f\nCalifornia1\x150\x13\x06\x03U\x04\a\f\fSanta Monica1\x110\x0F\x06\x03U\x04\n\f\bOneLogin1\x190\x17\x06\x03U\x04\x03\f\x10app.onelogin.com0\x81\x9F0\r\x06\t*\x86H\x86\xF7\r\x01\x01\x01\x05\x00\x03\x81\x8D\x000\x81\x89\x02\x81\x81\x00\xE8\xD2\xBBW\xE3?/\x1D\xE7\x0E\x10\xC8\xBD~\xCD\xDE!#\rL\x92G\xDF\xE1f?L\xB1\xBC9\x99\x14\xE5\x84\xD2Zi\x87<>d\xBD\x81\xF9\xBA\x85\xD2\xFF\xAA\x90\xF3Z\x97\xA5\x1D\xB0W\xC0\x93\xA3\x06IP\xB84\xF5\xD7Qu\x19\xFCB\xCA\xA3\xD4\\\x8E\v\x9B%\x13|\xB6m\x9D\xA8\x16\xE6\xBB\xDA\x87\xFF\xE3\xD7\xE9\xBA9\xC5O\xA2\xA7C\xADB\x04\xCA\xA5\x0E\x84\xD0\xA8\xE4\xFA\xDA\xF1\x89\xF2s\xFA1\x95\xAF\x03\xAB1\xAA\xE7y\x02\x03\x01\x00\x010\x03\x06\x01\x00\x03\x01\x00"
     assert cert = OpenSSL::X509::Certificate.new(cert_text)