package libp2pquic import ( "bytes" "context" "crypto/rand" "errors" "fmt" "io" mrand "math/rand" "net" "sync/atomic" "testing" "time" ic "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/network" mocknetwork "github.com/libp2p/go-libp2p/core/network/mocks" "github.com/libp2p/go-libp2p/core/peer" tpt "github.com/libp2p/go-libp2p/core/transport" "github.com/libp2p/go-libp2p/p2p/transport/quicreuse" ma "github.com/multiformats/go-multiaddr" "github.com/quic-go/quic-go" quicproxy "github.com/quic-go/quic-go/integrationtests/tools/proxy" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" ) //go:generate sh -c "go run go.uber.org/mock/mockgen -package libp2pquic -destination mock_connection_gater_test.go github.com/libp2p/go-libp2p/core/connmgr ConnectionGater && go run golang.org/x/tools/cmd/goimports -w mock_connection_gater_test.go" type connTestCase struct { Name string Options []quicreuse.Option } var connTestCases = []*connTestCase{ {"reuseport_on", []quicreuse.Option{}}, {"reuseport_off", []quicreuse.Option{quicreuse.DisableReuseport()}}, } func tStringCast(str string) ma.Multiaddr { m, _ := ma.StringCast(str) return m } func createPeer(t *testing.T) (peer.ID, ic.PrivKey) { var priv ic.PrivKey var err error switch mrand.Int() % 4 { case 0: priv, _, err = ic.GenerateECDSAKeyPair(rand.Reader) case 1: priv, _, err = ic.GenerateRSAKeyPair(2048, rand.Reader) case 2: priv, _, err = ic.GenerateEd25519Key(rand.Reader) case 3: priv, _, err = ic.GenerateSecp256k1Key(rand.Reader) } require.NoError(t, err) id, err := peer.IDFromPrivateKey(priv) require.NoError(t, err) t.Logf("using a %s key: %s", priv.Type(), id) return id, priv } func runServer(t *testing.T, tr tpt.Transport, addr string) tpt.Listener { t.Helper() ln, err := tr.Listen(tStringCast(addr)) require.NoError(t, err) return ln } func newConnManager(t *testing.T, opts ...quicreuse.Option) *quicreuse.ConnManager { t.Helper() cm, err := quicreuse.NewConnManager(quic.StatelessResetKey{}, quic.TokenGeneratorKey{}, opts...) require.NoError(t, err) t.Cleanup(func() { cm.Close() }) return cm } func TestHandshake(t *testing.T) { for _, tc := range connTestCases { t.Run(tc.Name, func(t *testing.T) { testHandshake(t, tc) }) } } func testHandshake(t *testing.T, tc *connTestCase) { serverID, serverKey := createPeer(t) clientID, clientKey := createPeer(t) serverTransport, err := NewTransport(serverKey, newConnManager(t, tc.Options...), nil, nil, nil) require.NoError(t, err) defer serverTransport.(io.Closer).Close() handshake := func(t *testing.T, ln tpt.Listener) { clientTransport, err := NewTransport(clientKey, newConnManager(t, tc.Options...), nil, nil, nil) require.NoError(t, err) defer clientTransport.(io.Closer).Close() conn, err := clientTransport.Dial(context.Background(), ln.Multiaddr(), serverID) require.NoError(t, err) defer conn.Close() serverConn, err := ln.Accept() require.NoError(t, err) defer serverConn.Close() require.Equal(t, conn.LocalPeer(), clientID) require.Equal(t, conn.RemotePeer(), serverID) require.True(t, conn.RemotePublicKey().Equals(serverKey.GetPublic()), "remote public key doesn't match") require.Equal(t, serverConn.LocalPeer(), serverID) require.Equal(t, serverConn.RemotePeer(), clientID) require.True(t, serverConn.RemotePublicKey().Equals(clientKey.GetPublic()), "remote public key doesn't match") } t.Run("on IPv4", func(t *testing.T) { ln := runServer(t, serverTransport, "/ip4/127.0.0.1/udp/0/quic-v1") defer ln.Close() handshake(t, ln) }) t.Run("on IPv6", func(t *testing.T) { ln := runServer(t, serverTransport, "/ip6/::1/udp/0/quic-v1") defer ln.Close() handshake(t, ln) }) } func TestResourceManagerSuccess(t *testing.T) { for _, tc := range connTestCases { t.Run(tc.Name, func(t *testing.T) { testResourceManagerSuccess(t, tc) }) } } func testResourceManagerSuccess(t *testing.T, tc *connTestCase) { serverID, serverKey := createPeer(t) clientID, clientKey := createPeer(t) ctrl := gomock.NewController(t) defer ctrl.Finish() serverRcmgr := mocknetwork.NewMockResourceManager(ctrl) serverTransport, err := NewTransport(serverKey, newConnManager(t, tc.Options...), nil, nil, serverRcmgr) require.NoError(t, err) defer serverTransport.(io.Closer).Close() ln, err := serverTransport.Listen(tStringCast("/ip4/127.0.0.1/udp/0/quic-v1")) require.NoError(t, err) defer ln.Close() clientRcmgr := mocknetwork.NewMockResourceManager(ctrl) clientTransport, err := NewTransport(clientKey, newConnManager(t, tc.Options...), nil, nil, clientRcmgr) require.NoError(t, err) defer clientTransport.(io.Closer).Close() connChan := make(chan tpt.CapableConn) serverConnScope := mocknetwork.NewMockConnManagementScope(ctrl) go func() { serverRcmgr.EXPECT().OpenConnection(network.DirInbound, false, gomock.Not(ln.Multiaddr())).Return(serverConnScope, nil) serverConnScope.EXPECT().SetPeer(clientID) serverConn, err := ln.Accept() require.NoError(t, err) connChan <- serverConn }() connScope := mocknetwork.NewMockConnManagementScope(ctrl) clientRcmgr.EXPECT().OpenConnection(network.DirOutbound, false, ln.Multiaddr()).Return(connScope, nil) connScope.EXPECT().SetPeer(serverID) conn, err := clientTransport.Dial(context.Background(), ln.Multiaddr(), serverID) require.NoError(t, err) serverConn := <-connChan t.Log("received conn") connScope.EXPECT().Done().MinTimes(1) // for dialed connections, we might call Done multiple times conn.Close() serverConnScope.EXPECT().Done() serverConn.Close() } func TestResourceManagerDialDenied(t *testing.T) { for _, tc := range connTestCases { t.Run(tc.Name, func(t *testing.T) { testResourceManagerDialDenied(t, tc) }) } } func testResourceManagerDialDenied(t *testing.T, tc *connTestCase) { _, clientKey := createPeer(t) ctrl := gomock.NewController(t) defer ctrl.Finish() rcmgr := mocknetwork.NewMockResourceManager(ctrl) clientTransport, err := NewTransport(clientKey, newConnManager(t, tc.Options...), nil, nil, rcmgr) require.NoError(t, err) defer clientTransport.(io.Closer).Close() connScope := mocknetwork.NewMockConnManagementScope(ctrl) target := tStringCast("/ip4/127.0.0.1/udp/1234/quic-v1") rcmgr.EXPECT().OpenConnection(network.DirOutbound, false, target).Return(connScope, nil) rerr := errors.New("nope") p := peer.ID("server") connScope.EXPECT().SetPeer(p).Return(rerr) connScope.EXPECT().Done() _, err = clientTransport.Dial(context.Background(), target, p) require.ErrorIs(t, err, rerr) } func TestResourceManagerAcceptDenied(t *testing.T) { for _, tc := range connTestCases { t.Run(tc.Name, func(t *testing.T) { testResourceManagerAcceptDenied(t, tc) }) } } func testResourceManagerAcceptDenied(t *testing.T, tc *connTestCase) { serverID, serverKey := createPeer(t) clientID, clientKey := createPeer(t) ctrl := gomock.NewController(t) defer ctrl.Finish() clientRcmgr := mocknetwork.NewMockResourceManager(ctrl) clientTransport, err := NewTransport(clientKey, newConnManager(t, tc.Options...), nil, nil, clientRcmgr) require.NoError(t, err) defer clientTransport.(io.Closer).Close() serverRcmgr := mocknetwork.NewMockResourceManager(ctrl) serverConnScope := mocknetwork.NewMockConnManagementScope(ctrl) rerr := errors.New("denied") gomock.InOrder( serverRcmgr.EXPECT().OpenConnection(network.DirInbound, false, gomock.Any()).Return(serverConnScope, nil), serverConnScope.EXPECT().SetPeer(clientID).Return(rerr), serverConnScope.EXPECT().Done(), ) serverTransport, err := NewTransport(serverKey, newConnManager(t, tc.Options...), nil, nil, serverRcmgr) require.NoError(t, err) defer serverTransport.(io.Closer).Close() ln, err := serverTransport.Listen(tStringCast("/ip4/127.0.0.1/udp/0/quic-v1")) require.NoError(t, err) defer ln.Close() connChan := make(chan tpt.CapableConn) go func() { ln.Accept() close(connChan) }() clientConnScope := mocknetwork.NewMockConnManagementScope(ctrl) clientRcmgr.EXPECT().OpenConnection(network.DirOutbound, false, ln.Multiaddr()).Return(clientConnScope, nil) clientConnScope.EXPECT().SetPeer(serverID) // In rare instances, the connection gating error will already occur on Dial. // In that case, Done is called on the connection scope. clientConnScope.EXPECT().Done().MaxTimes(1) conn, err := clientTransport.Dial(context.Background(), ln.Multiaddr(), serverID) // In rare instances, the connection gating error will already occur on Dial. if err == nil { _, err = conn.AcceptStream() require.Error(t, err) } select { case <-connChan: t.Fatal("didn't expect to accept a connection") default: } } func TestStreams(t *testing.T) { for _, tc := range connTestCases { t.Run(tc.Name, func(t *testing.T) { testStreams(t, tc) }) } } func testStreams(t *testing.T, tc *connTestCase) { serverID, serverKey := createPeer(t) _, clientKey := createPeer(t) serverTransport, err := NewTransport(serverKey, newConnManager(t, tc.Options...), nil, nil, nil) require.NoError(t, err) defer serverTransport.(io.Closer).Close() ln := runServer(t, serverTransport, "/ip4/127.0.0.1/udp/0/quic-v1") defer ln.Close() clientTransport, err := NewTransport(clientKey, newConnManager(t, tc.Options...), nil, nil, nil) require.NoError(t, err) defer clientTransport.(io.Closer).Close() conn, err := clientTransport.Dial(context.Background(), ln.Multiaddr(), serverID) require.NoError(t, err) defer conn.Close() serverConn, err := ln.Accept() require.NoError(t, err) defer serverConn.Close() str, err := conn.OpenStream(context.Background()) require.NoError(t, err) _, err = str.Write([]byte("foobar")) require.NoError(t, err) str.Close() sstr, err := serverConn.AcceptStream() require.NoError(t, err) data, err := io.ReadAll(sstr) require.NoError(t, err) require.Equal(t, data, []byte("foobar")) } func TestHandshakeFailPeerIDMismatch(t *testing.T) { for _, tc := range connTestCases { t.Run(tc.Name, func(t *testing.T) { testHandshakeFailPeerIDMismatch(t, tc) }) } } func testHandshakeFailPeerIDMismatch(t *testing.T, tc *connTestCase) { _, serverKey := createPeer(t) _, clientKey := createPeer(t) thirdPartyID, _ := createPeer(t) serverTransport, err := NewTransport(serverKey, newConnManager(t, tc.Options...), nil, nil, nil) require.NoError(t, err) defer serverTransport.(io.Closer).Close() ln := runServer(t, serverTransport, "/ip4/127.0.0.1/udp/0/quic-v1") clientTransport, err := NewTransport(clientKey, newConnManager(t, tc.Options...), nil, nil, nil) require.NoError(t, err) // dial, but expect the wrong peer ID _, err = clientTransport.Dial(context.Background(), ln.Multiaddr(), thirdPartyID) require.Error(t, err) require.Contains(t, err.Error(), "CRYPTO_ERROR") defer clientTransport.(io.Closer).Close() acceptErr := make(chan error) go func() { _, err := ln.Accept() acceptErr <- err }() select { case <-acceptErr: t.Fatal("didn't expect Accept to return before being closed") case <-time.After(100 * time.Millisecond): } require.NoError(t, ln.Close()) require.Error(t, <-acceptErr) } func TestConnectionGating(t *testing.T) { for _, tc := range connTestCases { t.Run(tc.Name, func(t *testing.T) { testConnectionGating(t, tc) }) } } func testConnectionGating(t *testing.T, tc *connTestCase) { serverID, serverKey := createPeer(t) _, clientKey := createPeer(t) mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() cg := NewMockConnectionGater(mockCtrl) t.Run("accepted connections", func(t *testing.T) { serverTransport, err := NewTransport(serverKey, newConnManager(t, tc.Options...), nil, cg, nil) defer serverTransport.(io.Closer).Close() require.NoError(t, err) ln := runServer(t, serverTransport, "/ip4/127.0.0.1/udp/0/quic-v1") defer ln.Close() cg.EXPECT().InterceptAccept(gomock.Any()) accepted := make(chan struct{}) go func() { defer close(accepted) _, err := ln.Accept() require.NoError(t, err) }() clientTransport, err := NewTransport(clientKey, newConnManager(t, tc.Options...), nil, nil, nil) require.NoError(t, err) defer clientTransport.(io.Closer).Close() // make sure that connection attempts fails conn, err := clientTransport.Dial(context.Background(), ln.Multiaddr(), serverID) // In rare instances, the connection gating error will already occur on Dial. // In most cases, it will be returned by AcceptStream. if err == nil { _, err = conn.AcceptStream() } require.Contains(t, err.Error(), "connection gated") // now allow the address and make sure the connection goes through cg.EXPECT().InterceptAccept(gomock.Any()).Return(true) cg.EXPECT().InterceptSecured(gomock.Any(), gomock.Any(), gomock.Any()).Return(true) conn, err = clientTransport.Dial(context.Background(), ln.Multiaddr(), serverID) require.NoError(t, err) defer conn.Close() require.Eventually(t, func() bool { select { case <-accepted: return true default: return false } }, time.Second, 10*time.Millisecond) }) t.Run("secured connections", func(t *testing.T) { serverTransport, err := NewTransport(serverKey, newConnManager(t, tc.Options...), nil, nil, nil) require.NoError(t, err) defer serverTransport.(io.Closer).Close() ln := runServer(t, serverTransport, "/ip4/127.0.0.1/udp/0/quic-v1") defer ln.Close() cg := NewMockConnectionGater(mockCtrl) cg.EXPECT().InterceptSecured(gomock.Any(), gomock.Any(), gomock.Any()) clientTransport, err := NewTransport(clientKey, newConnManager(t, tc.Options...), nil, cg, nil) require.NoError(t, err) defer clientTransport.(io.Closer).Close() // make sure that connection attempts fails _, err = clientTransport.Dial(context.Background(), ln.Multiaddr(), serverID) require.Error(t, err) require.Contains(t, err.Error(), "connection gated") // now allow the peerId and make sure the connection goes through cg.EXPECT().InterceptSecured(gomock.Any(), gomock.Any(), gomock.Any()).Return(true) conn, err := clientTransport.Dial(context.Background(), ln.Multiaddr(), serverID) require.NoError(t, err) conn.Close() }) } func TestDialTwo(t *testing.T) { for _, tc := range connTestCases { t.Run(tc.Name, func(t *testing.T) { testDialTwo(t, tc) }) } } func testDialTwo(t *testing.T, tc *connTestCase) { serverID, serverKey := createPeer(t) _, clientKey := createPeer(t) serverID2, serverKey2 := createPeer(t) serverTransport, err := NewTransport(serverKey, newConnManager(t, tc.Options...), nil, nil, nil) require.NoError(t, err) defer serverTransport.(io.Closer).Close() ln1 := runServer(t, serverTransport, "/ip4/127.0.0.1/udp/0/quic-v1") defer ln1.Close() serverTransport2, err := NewTransport(serverKey2, newConnManager(t, tc.Options...), nil, nil, nil) require.NoError(t, err) defer serverTransport2.(io.Closer).Close() ln2 := runServer(t, serverTransport2, "/ip4/127.0.0.1/udp/0/quic-v1") defer ln2.Close() data := bytes.Repeat([]byte{'a'}, 5*1<<20) // 5 MB // wait for both servers to accept a connection // then send some data go func() { serverConn1, err := ln1.Accept() require.NoError(t, err) serverConn2, err := ln2.Accept() require.NoError(t, err) for _, c := range []tpt.CapableConn{serverConn1, serverConn2} { go func(conn tpt.CapableConn) { str, err := conn.OpenStream(context.Background()) require.NoError(t, err) defer str.Close() _, err = str.Write(data) require.NoError(t, err) }(c) } }() clientTransport, err := NewTransport(clientKey, newConnManager(t, tc.Options...), nil, nil, nil) require.NoError(t, err) defer clientTransport.(io.Closer).Close() c1, err := clientTransport.Dial(context.Background(), ln1.Multiaddr(), serverID) require.NoError(t, err) defer c1.Close() c2, err := clientTransport.Dial(context.Background(), ln2.Multiaddr(), serverID2) require.NoError(t, err) defer c2.Close() done := make(chan struct{}, 2) // receive the data on both connections at the same time for _, c := range []tpt.CapableConn{c1, c2} { go func(conn tpt.CapableConn) { str, err := conn.AcceptStream() require.NoError(t, err) str.CloseWrite() d, err := io.ReadAll(str) require.NoError(t, err) require.Equal(t, d, data) done <- struct{}{} }(c) } for i := 0; i < 2; i++ { require.Eventually(t, func() bool { select { case <-done: return true default: return false } }, 15*time.Second, 50*time.Millisecond) } } func TestStatelessReset(t *testing.T) { for _, tc := range connTestCases { t.Run(tc.Name, func(t *testing.T) { testStatelessReset(t, tc) }) } } func testStatelessReset(t *testing.T, tc *connTestCase) { serverID, serverKey := createPeer(t) _, clientKey := createPeer(t) serverTransport, err := NewTransport(serverKey, newConnManager(t, tc.Options...), nil, nil, nil) require.NoError(t, err) defer serverTransport.(io.Closer).Close() ln := runServer(t, serverTransport, "/ip4/127.0.0.1/udp/0/quic-v1") var drop uint32 dropCallback := func(quicproxy.Direction, []byte) bool { return atomic.LoadUint32(&drop) > 0 } proxy, err := quicproxy.NewQuicProxy("localhost:0", &quicproxy.Opts{ RemoteAddr: fmt.Sprintf("localhost:%d", ln.Addr().(*net.UDPAddr).Port), DropPacket: dropCallback, }) require.NoError(t, err) proxyLocalAddr := proxy.LocalAddr() // establish a connection clientTransport, err := NewTransport(clientKey, newConnManager(t, tc.Options...), nil, nil, nil) require.NoError(t, err) defer clientTransport.(io.Closer).Close() proxyAddr, err := quicreuse.ToQuicMultiaddr(proxy.LocalAddr(), quic.Version1) require.NoError(t, err) conn, err := clientTransport.Dial(context.Background(), proxyAddr, serverID) require.NoError(t, err) connChan := make(chan tpt.CapableConn) go func() { conn, err := ln.Accept() require.NoError(t, err) str, err := conn.OpenStream(context.Background()) require.NoError(t, err) _, err = conn.LocalMultiaddr().ValueForProtocol(ma.P_QUIC_V1) require.NoError(t, err) str.Write([]byte("foobar")) connChan <- conn }() str, err := conn.AcceptStream() require.NoError(t, err) _, err = str.Read(make([]byte, 6)) require.NoError(t, err) // Stop forwarding packets and close the server. // This prevents the CONNECTION_CLOSE from reaching the client. atomic.StoreUint32(&drop, 1) ln.Close() (<-connChan).Close() proxy.Close() // Start another listener (on a different port). ln, err = serverTransport.Listen(tStringCast("/ip4/127.0.0.1/udp/0/quic-v1")) require.NoError(t, err) defer ln.Close() // Now that the new server is up, re-enable packet forwarding. atomic.StoreUint32(&drop, 0) // Recreate the proxy, such that its client-facing port stays constant. proxy, err = quicproxy.NewQuicProxy(proxyLocalAddr.String(), &quicproxy.Opts{ RemoteAddr: fmt.Sprintf("localhost:%d", ln.Addr().(*net.UDPAddr).Port), DropPacket: dropCallback, }) require.NoError(t, err) defer proxy.Close() // Trigger something (not too small) to be sent, so that we receive the stateless reset. // The new server doesn't have any state for the previously established connection. // We expect it to send a stateless reset. _, rerr := str.Write([]byte("Lorem ipsum dolor sit amet.")) if rerr == nil { _, rerr = str.Read([]byte{0, 0}) } require.Error(t, rerr) var statelessResetErr *quic.StatelessResetError require.ErrorAs(t, rerr, &statelessResetErr) } // Hole punching is only expected to work with reuseport enabled. // We don't need to test `DisableReuseport` option. func TestHolePunching(t *testing.T) { serverID, serverKey := createPeer(t) clientID, clientKey := createPeer(t) t1, err := NewTransport(serverKey, newConnManager(t), nil, nil, nil) require.NoError(t, err) defer t1.(io.Closer).Close() laddr, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/0/quic-v1") require.NoError(t, err) ln1, err := t1.Listen(laddr) require.NoError(t, err) done1 := make(chan struct{}) go func() { defer close(done1) _, err := ln1.Accept() require.Error(t, err, "didn't expect to accept any connections") }() t2, err := NewTransport(clientKey, newConnManager(t), nil, nil, nil) require.NoError(t, err) defer t2.(io.Closer).Close() ln2, err := t2.Listen(laddr) require.NoError(t, err) done2 := make(chan struct{}) go func() { defer close(done2) _, err := ln2.Accept() require.Error(t, err, "didn't expect to accept any connections") }() connChan := make(chan tpt.CapableConn) go func() { conn, err := t2.Dial( network.WithSimultaneousConnect(context.Background(), false, ""), ln1.Multiaddr(), serverID, ) require.NoError(t, err) connChan <- conn }() // Make sure the server role (the dial on t2) has progressed far enough. // If it hasn't created the hole punch map entry, the connection will be accepted as a regular connection, // which would make this test fail. require.Eventually(t, func() bool { tr := t2.(*transport) tr.holePunchingMx.Lock() defer tr.holePunchingMx.Unlock() return len(tr.holePunching) > 0 }, time.Second, 10*time.Millisecond) conn1, err := t1.Dial( network.WithSimultaneousConnect(context.Background(), true, ""), ln2.Multiaddr(), clientID, ) require.NoError(t, err) defer conn1.Close() require.Equal(t, conn1.RemotePeer(), clientID) var conn2 tpt.CapableConn require.Eventually(t, func() bool { select { case conn2 = <-connChan: return true default: return false } }, time.Second, 10*time.Millisecond) defer conn2.Close() require.Equal(t, conn2.RemotePeer(), serverID) ln1.Close() ln2.Close() <-done1 <-done2 }