[3.3] Add http2 client connection preface process (#15436)

* Add http2 client connection preface process

* Add comments to explain the reason of adding client connection preface processing

Author: zrlw

dubbo-remoting/dubbo-remoting-netty4/src/main/java/org/apache/dubbo/remoting/transport/netty4/NettyConnectionClient.java

@@ -17,31 +17,45 @@
 package org.apache.dubbo.remoting.transport.netty4;
 
 import org.apache.dubbo.common.URL;
+import org.apache.dubbo.common.Version;
+import org.apache.dubbo.common.utils.NetUtils;
 import org.apache.dubbo.remoting.ChannelHandler;
 import org.apache.dubbo.remoting.Constants;
 import org.apache.dubbo.remoting.RemotingException;
 import org.apache.dubbo.remoting.api.WireProtocol;
+import org.apache.dubbo.remoting.transport.netty4.http2.Http2ClientSettingsHandler;
 import org.apache.dubbo.remoting.transport.netty4.ssl.SslClientTlsHandler;
 import org.apache.dubbo.remoting.transport.netty4.ssl.SslContexts;
 import org.apache.dubbo.remoting.utils.UrlUtils;
 
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
 import io.netty.bootstrap.Bootstrap;
 import io.netty.buffer.PooledByteBufAllocator;
 import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelInitializer;
 import io.netty.channel.ChannelOption;
 import io.netty.channel.ChannelPipeline;
 import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.codec.http2.Http2FrameCodec;
 import io.netty.handler.ssl.SslContext;
 import io.netty.handler.timeout.IdleStateHandler;
+import io.netty.util.concurrent.DefaultPromise;
+import io.netty.util.concurrent.GlobalEventExecutor;
+import io.netty.util.concurrent.Promise;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.dubbo.common.constants.LoggerCodeConstants.TRANSPORT_CLIENT_CONNECT_TIMEOUT;
 import static org.apache.dubbo.remoting.transport.netty4.NettyEventLoopFactory.socketChannelClass;
 
 public final class NettyConnectionClient extends AbstractNettyConnectionClient {
 
     private Bootstrap bootstrap;
 
+    private AtomicReference<Promise<Void>> connectionPrefaceReceivedPromiseRef;
+
     public NettyConnectionClient(URL url, ChannelHandler handler) throws RemotingException {
         super(url, handler);
     }
@@ -87,6 +101,16 @@ protected void initChannel(SocketChannel ch) {
 
                 NettyConfigOperator operator = new NettyConfigOperator(nettyChannel, getChannelHandler());
                 protocol.configClientPipeline(getUrl(), operator, nettySslContextOperator);
+
+                ChannelHandlerContext http2FrameCodecHandlerCtx = pipeline.context(Http2FrameCodec.class);
+                if (http2FrameCodecHandlerCtx != null) {
+                    connectionPrefaceReceivedPromiseRef = new AtomicReference<>();
+                    pipeline.addAfter(
+                            http2FrameCodecHandlerCtx.name(),
+                            "client-connection-preface-handler",
+                            new Http2ClientSettingsHandler(connectionPrefaceReceivedPromiseRef));
+                }
+
                 // set null but do not close this client, it will be reconnecting in the future
                 ch.closeFuture().addListener(channelFuture -> clearNettyChannel());
                 // TODO support Socks5
@@ -97,6 +121,71 @@ protected void initChannel(SocketChannel ch) {
 
     @Override
     protected ChannelFuture performConnect() {
+        if (connectionPrefaceReceivedPromiseRef != null) {
+            connectionPrefaceReceivedPromiseRef.compareAndSet(null, new DefaultPromise<>(GlobalEventExecutor.INSTANCE));
+        }
         return bootstrap.connect();
     }
+
+    @Override
+    protected void doConnect() throws RemotingException {
+        long start = System.currentTimeMillis();
+        super.doConnect();
+        waitConnectionPreface(start);
+    }
+
+    /**
+     * Wait connection preface
+     * <br>
+     * Http2 client should set max header list size of http2 encoder based on server connection preface before
+     * sending first data frame, otherwise the http2 server might send back GO_AWAY frame and disconnect the connection
+     * immediately if the size of client Headers frame is bigger than the MAX_HEADER_LIST_SIZE of server settings.<br>
+     * @see <a href="https://httpwg.org/specs/rfc7540.html#ConnectionHeader">HTTP/2 Connection Preface</a><br>
+     * In HTTP/2, each endpoint is required to send a connection preface as a final confirmation of the protocol
+     * in use and to establish the initial settings for the HTTP/2 connection. The client and server each send a
+     * different connection preface. The client connection preface starts with a sequence of 24 octets,
+     * which in hex notation is:<br>
+     * 0x505249202a20485454502f322e300d0a0d0a534d0d0a0d0a<br>
+     * That is, the connection preface starts with the string PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n<br>
+     * This sequence MUST be followed by a SETTINGS frame (Section 6.5), which MAY be empty.
+     * The server connection preface consists of a potentially empty SETTINGS frame (Section 6.5) that MUST be
+     * the first frame the server sends in the HTTP/2 connection.
+     *
+     * @param start start time of doConnect in milliseconds.
+     */
+    private void waitConnectionPreface(long start) throws RemotingException {
+        if (connectionPrefaceReceivedPromiseRef == null) {
+            return;
+        }
+        Promise<Void> connectionPrefaceReceivedPromise = connectionPrefaceReceivedPromiseRef.get();
+        if (connectionPrefaceReceivedPromise != null) {
+            long retainedTimeout = getConnectTimeout() - System.currentTimeMillis() + start;
+            boolean ret = connectionPrefaceReceivedPromise.awaitUninterruptibly(retainedTimeout, TimeUnit.MILLISECONDS);
+            // Only process once: destroy connectionPrefaceReceivedPromise after used
+            synchronized (this) {
+                connectionPrefaceReceivedPromiseRef.set(null);
+            }
+            if (!ret || !connectionPrefaceReceivedPromise.isSuccess()) {
+                // 6-2 Client-side connection preface timeout
+                RemotingException remotingException = new RemotingException(
+                        this,
+                        "client(url: " + getUrl() + ") failed to connect to server " + getConnectAddress()
+                                + " client-side connection preface timeout " + getConnectTimeout()
+                                + "ms (elapsed: "
+                                + (System.currentTimeMillis() - start) + "ms) from netty client "
+                                + NetUtils.getLocalHost()
+                                + " using dubbo version "
+                                + Version.getVersion());
+
+                logger.error(
+                        TRANSPORT_CLIENT_CONNECT_TIMEOUT,
+                        "provider crash",
+                        "",
+                        "Client-side connection preface timeout",
+                        remotingException);
+
+                throw remotingException;
+            }
+        }
+    }
 }

dubbo-remoting/dubbo-remoting-netty4/src/main/java/org/apache/dubbo/remoting/transport/netty4/http2/Http2ClientSettingsHandler.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.remoting.transport.netty4.http2;
+
+import org.apache.dubbo.common.logger.Logger;
+import org.apache.dubbo.common.logger.LoggerFactory;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.handler.codec.http2.Http2SettingsFrame;
+import io.netty.util.concurrent.Promise;
+
+public class Http2ClientSettingsHandler extends SimpleChannelInboundHandler<Http2SettingsFrame> {
+
+    private static final Logger logger = LoggerFactory.getLogger(Http2ClientSettingsHandler.class);
+
+    private final AtomicReference<Promise<Void>> connectionPrefaceReceivedPromiseRef;
+
+    public Http2ClientSettingsHandler(AtomicReference<Promise<Void>> connectionPrefaceReceivedPromiseRef) {
+        this.connectionPrefaceReceivedPromiseRef = connectionPrefaceReceivedPromiseRef;
+    }
+
+    @Override
+    protected void channelRead0(ChannelHandlerContext ctx, Http2SettingsFrame msg) throws Exception {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Receive Http2 Settings frame of " + ctx.channel().localAddress() + " -> "
+                    + ctx.channel().remoteAddress());
+        }
+        // connectionPrefaceReceivedPromise will be set null after first used.
+        Promise<Void> connectionPrefaceReceivedPromise = connectionPrefaceReceivedPromiseRef.get();
+        if (connectionPrefaceReceivedPromise == null) {
+            ctx.fireChannelRead(msg);
+        } else {
+            // Notify the connection preface is received when first inbound http2 settings frame is arrived.
+            connectionPrefaceReceivedPromise.trySuccess(null);
+        }
+    }
+}