ceremonyclient/go-libp2p/p2p/protocol/identify/obsaddr.go

638 lines
19 KiB
Go

package identify
import (
"context"
"fmt"
"sync"
"time"
"golang.org/x/exp/slices"
"github.com/libp2p/go-libp2p/core/event"
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peerstore"
"github.com/libp2p/go-libp2p/p2p/host/eventbus"
ma "github.com/multiformats/go-multiaddr"
manet "github.com/multiformats/go-multiaddr/net"
)
// ActivationThresh sets how many times an address must be seen as "activated"
// and therefore advertised to other peers as an address that the local peer
// can be contacted on. The "seen" events expire by default after 40 minutes
// (OwnObservedAddressTTL * ActivationThreshold). The are cleaned up during
// the GC rounds set by GCInterval.
var ActivationThresh = 4
// GCInterval specicies how often to make a round cleaning seen events and
// observed addresses. An address will be cleaned if it has not been seen in
// OwnObservedAddressTTL (10 minutes). A "seen" event will be cleaned up if
// it is older than OwnObservedAddressTTL * ActivationThresh (40 minutes).
var GCInterval = 10 * time.Minute
// observedAddrManagerWorkerChannelSize defines how many addresses can be enqueued
// for adding to an ObservedAddrManager.
var observedAddrManagerWorkerChannelSize = 16
// maxObservedAddrsPerIPAndTransport is the maximum number of observed addresses
// we will return for each (IPx/TCP or UDP) group.
var maxObservedAddrsPerIPAndTransport = 2
// observation records an address observation from an "observer" (where every IP
// address is a unique observer).
type observation struct {
// seenTime is the last time this observation was made.
seenTime time.Time
// inbound indicates whether or not this observation has been made from
// an inbound connection. This remains true even if we an observation
// from a subsequent outbound connection.
inbound bool
}
// observedAddr is an entry for an address reported by our peers.
// We only use addresses that:
// - have been observed at least 4 times in last 40 minutes. (counter symmetric nats)
// - have been observed at least once recently (10 minutes), because our position in the
// network, or network port mapppings, may have changed.
type observedAddr struct {
addr ma.Multiaddr
seenBy map[string]observation // peer(observer) address -> observation info
lastSeen time.Time
numInbound int
}
func (oa *observedAddr) activated() bool {
// We only activate if other peers observed the same address
// of ours at least 4 times. SeenBy peers are removed by GC if
// they say the address more than ttl*ActivationThresh
return len(oa.seenBy) >= ActivationThresh
}
// GroupKey returns the group in which this observation belongs. Currently, an
// observed address's group is just the address with all ports set to 0. This
// means we can advertise the most commonly observed external ports without
// advertising _every_ observed port.
func (oa *observedAddr) groupKey() string {
key := make([]byte, 0, len(oa.addr.Bytes()))
ma.ForEach(oa.addr, func(c ma.Component) bool {
switch proto := c.Protocol(); proto.Code {
case ma.P_TCP, ma.P_UDP:
key = append(key, proto.VCode...)
key = append(key, 0, 0) // zero in two bytes
default:
key = append(key, c.Bytes()...)
}
return true
})
return string(key)
}
type newObservation struct {
conn network.Conn
observed ma.Multiaddr
}
// ObservedAddrManager keeps track of a ObservedAddrs.
type ObservedAddrManager struct {
host host.Host
closeOnce sync.Once
refCount sync.WaitGroup
ctx context.Context // the context is canceled when Close is called
ctxCancel context.CancelFunc
// latest observation from active connections
// we'll "re-observe" these when we gc
activeConnsMu sync.Mutex
// active connection -> most recent observation
activeConns map[network.Conn]ma.Multiaddr
mu sync.RWMutex
closed bool
// local(internal) address -> list of observed(external) addresses
addrs map[string][]*observedAddr
ttl time.Duration
refreshTimer *time.Timer
// this is the worker channel
wch chan newObservation
reachabilitySub event.Subscription
reachability network.Reachability
currentUDPNATDeviceType network.NATDeviceType
currentTCPNATDeviceType network.NATDeviceType
emitNATDeviceTypeChanged event.Emitter
}
// NewObservedAddrManager returns a new address manager using
// peerstore.OwnObservedAddressTTL as the TTL.
func NewObservedAddrManager(host host.Host) (*ObservedAddrManager, error) {
oas := &ObservedAddrManager{
addrs: make(map[string][]*observedAddr),
ttl: peerstore.OwnObservedAddrTTL,
wch: make(chan newObservation, observedAddrManagerWorkerChannelSize),
host: host,
activeConns: make(map[network.Conn]ma.Multiaddr),
// refresh every ttl/2 so we don't forget observations from connected peers
refreshTimer: time.NewTimer(peerstore.OwnObservedAddrTTL / 2),
}
oas.ctx, oas.ctxCancel = context.WithCancel(context.Background())
reachabilitySub, err := host.EventBus().Subscribe(new(event.EvtLocalReachabilityChanged), eventbus.Name("identify (obsaddr)"))
if err != nil {
return nil, fmt.Errorf("failed to subscribe to reachability event: %s", err)
}
oas.reachabilitySub = reachabilitySub
emitter, err := host.EventBus().Emitter(new(event.EvtNATDeviceTypeChanged), eventbus.Stateful)
if err != nil {
return nil, fmt.Errorf("failed to create emitter for NATDeviceType: %s", err)
}
oas.emitNATDeviceTypeChanged = emitter
oas.host.Network().Notify((*obsAddrNotifiee)(oas))
oas.refCount.Add(1)
go oas.worker()
return oas, nil
}
// AddrsFor return all activated observed addresses associated with the given
// (resolved) listen address.
func (oas *ObservedAddrManager) AddrsFor(addr ma.Multiaddr) (addrs []ma.Multiaddr) {
oas.mu.RLock()
defer oas.mu.RUnlock()
if len(oas.addrs) == 0 {
return nil
}
observedAddrs, ok := oas.addrs[string(addr.Bytes())]
if !ok {
return
}
return oas.filter(observedAddrs)
}
// Addrs return all activated observed addresses
func (oas *ObservedAddrManager) Addrs() []ma.Multiaddr {
oas.mu.RLock()
defer oas.mu.RUnlock()
if len(oas.addrs) == 0 {
return nil
}
var allObserved []*observedAddr
for _, addrs := range oas.addrs {
allObserved = append(allObserved, addrs...)
}
return oas.filter(allObserved)
}
func (oas *ObservedAddrManager) filter(observedAddrs []*observedAddr) []ma.Multiaddr {
pmap := make(map[string][]*observedAddr)
now := time.Now()
for i := range observedAddrs {
a := observedAddrs[i]
if now.Sub(a.lastSeen) <= oas.ttl && a.activated() {
// group addresses by their IPX/Transport Protocol(TCP or UDP) pattern.
pat := a.groupKey()
pmap[pat] = append(pmap[pat], a)
}
}
addrs := make([]ma.Multiaddr, 0, len(observedAddrs))
for pat := range pmap {
s := pmap[pat]
// We prefer inbound connection observations over outbound.
// For ties, we prefer the ones with more votes.
slices.SortFunc(s, func(first, second *observedAddr) int {
if first.numInbound > second.numInbound {
return 1
}
return len(first.seenBy) - len(second.seenBy)
})
for i := 0; i < maxObservedAddrsPerIPAndTransport && i < len(s); i++ {
addrs = append(addrs, s[i].addr)
}
}
return addrs
}
// Record records an address observation, if valid.
func (oas *ObservedAddrManager) Record(conn network.Conn, observed ma.Multiaddr) {
select {
case oas.wch <- newObservation{
conn: conn,
observed: observed,
}:
default:
log.Debugw("dropping address observation due to full buffer",
"from", conn.RemoteMultiaddr(),
"observed", observed,
)
}
}
func (oas *ObservedAddrManager) worker() {
defer oas.refCount.Done()
ticker := time.NewTicker(GCInterval)
defer ticker.Stop()
subChan := oas.reachabilitySub.Out()
for {
select {
case evt, ok := <-subChan:
if !ok {
subChan = nil
continue
}
ev := evt.(event.EvtLocalReachabilityChanged)
oas.reachability = ev.Reachability
case obs := <-oas.wch:
oas.maybeRecordObservation(obs.conn, obs.observed)
case <-ticker.C:
oas.gc()
case <-oas.refreshTimer.C:
oas.refresh()
case <-oas.ctx.Done():
return
}
}
}
func (oas *ObservedAddrManager) refresh() {
oas.activeConnsMu.Lock()
recycledObservations := make([]newObservation, 0, len(oas.activeConns))
for conn, observed := range oas.activeConns {
recycledObservations = append(recycledObservations, newObservation{
conn: conn,
observed: observed,
})
}
oas.activeConnsMu.Unlock()
oas.mu.Lock()
defer oas.mu.Unlock()
for _, obs := range recycledObservations {
oas.recordObservationUnlocked(obs.conn, obs.observed)
}
// refresh every ttl/2 so we don't forget observations from connected peers
oas.refreshTimer.Reset(oas.ttl / 2)
}
func (oas *ObservedAddrManager) gc() {
oas.mu.Lock()
defer oas.mu.Unlock()
now := time.Now()
for local, observedAddrs := range oas.addrs {
filteredAddrs := observedAddrs[:0]
for _, a := range observedAddrs {
// clean up SeenBy set
for k, ob := range a.seenBy {
if now.Sub(ob.seenTime) > oas.ttl*time.Duration(ActivationThresh) {
delete(a.seenBy, k)
if ob.inbound {
a.numInbound--
}
}
}
// leave only alive observed addresses
if now.Sub(a.lastSeen) <= oas.ttl {
filteredAddrs = append(filteredAddrs, a)
}
}
if len(filteredAddrs) > 0 {
oas.addrs[local] = filteredAddrs
} else {
delete(oas.addrs, local)
}
}
}
func (oas *ObservedAddrManager) addConn(conn network.Conn, observed ma.Multiaddr) {
oas.activeConnsMu.Lock()
defer oas.activeConnsMu.Unlock()
// We need to make sure we haven't received a disconnect event for this
// connection yet. The only way to do that right now is to make sure the
// swarm still has the connection.
//
// Doing this under a lock that we _also_ take in a disconnect event
// handler ensures everything happens in the right order.
for _, c := range oas.host.Network().ConnsToPeer(conn.RemotePeer()) {
if c == conn {
oas.activeConns[conn] = observed
return
}
}
}
func (oas *ObservedAddrManager) removeConn(conn network.Conn) {
// DO NOT remove this lock.
// This ensures we don't call addConn at the same time:
// 1. see that we have a connection and pause inside addConn right before recording it.
// 2. process a disconnect event.
// 3. record the connection (leaking it).
oas.activeConnsMu.Lock()
delete(oas.activeConns, conn)
oas.activeConnsMu.Unlock()
}
type normalizeMultiaddrer interface {
NormalizeMultiaddr(addr ma.Multiaddr) ma.Multiaddr
}
type addrsProvider interface {
Addrs() []ma.Multiaddr
}
type listenAddrsProvider interface {
ListenAddresses() []ma.Multiaddr
InterfaceListenAddresses() ([]ma.Multiaddr, error)
}
func shouldRecordObservation(host addrsProvider, network listenAddrsProvider, conn network.ConnMultiaddrs, observed ma.Multiaddr) bool {
// First, determine if this observation is even worth keeping...
// Ignore observations from loopback nodes. We already know our loopback
// addresses.
if manet.IsIPLoopback(observed) {
return false
}
// Provided by NAT64 peers, these addresses are specific to the peer and not publicly routable
if manet.IsNAT64IPv4ConvertedIPv6Addr(observed) {
return false
}
// we should only use ObservedAddr when our connection's LocalAddr is one
// of our ListenAddrs. If we Dial out using an ephemeral addr, knowing that
// address's external mapping is not very useful because the port will not be
// the same as the listen addr.
ifaceaddrs, err := network.InterfaceListenAddresses()
if err != nil {
log.Infof("failed to get interface listen addrs", err)
return false
}
normalizer, canNormalize := host.(normalizeMultiaddrer)
if canNormalize {
for i, a := range ifaceaddrs {
ifaceaddrs[i] = normalizer.NormalizeMultiaddr(a)
}
}
local := conn.LocalMultiaddr()
if canNormalize {
local = normalizer.NormalizeMultiaddr(local)
}
listenAddrs := network.ListenAddresses()
if canNormalize {
for i, a := range listenAddrs {
listenAddrs[i] = normalizer.NormalizeMultiaddr(a)
}
}
if !ma.Contains(ifaceaddrs, local) && !ma.Contains(listenAddrs, local) {
// not in our list
return false
}
hostAddrs := host.Addrs()
if canNormalize {
for i, a := range hostAddrs {
hostAddrs[i] = normalizer.NormalizeMultiaddr(a)
}
}
// We should reject the connection if the observation doesn't match the
// transports of one of our advertised addresses.
if !HasConsistentTransport(observed, hostAddrs) &&
!HasConsistentTransport(observed, listenAddrs) {
log.Debugw(
"observed multiaddr doesn't match the transports of any announced addresses",
"from", conn.RemoteMultiaddr(),
"observed", observed,
)
return false
}
return true
}
func (oas *ObservedAddrManager) maybeRecordObservation(conn network.Conn, observed ma.Multiaddr) {
shouldRecord := shouldRecordObservation(oas.host, oas.host.Network(), conn, observed)
if shouldRecord {
// Ok, the observation is good, record it.
log.Debugw("added own observed listen addr", "observed", observed)
defer oas.addConn(conn, observed)
oas.mu.Lock()
defer oas.mu.Unlock()
oas.recordObservationUnlocked(conn, observed)
if oas.reachability == network.ReachabilityPrivate {
oas.emitAllNATTypes()
}
}
}
func (oas *ObservedAddrManager) recordObservationUnlocked(conn network.Conn, observed ma.Multiaddr) {
now := time.Now()
observerString := observerGroup(conn.RemoteMultiaddr())
localString := string(conn.LocalMultiaddr().Bytes())
ob := observation{
seenTime: now,
inbound: conn.Stat().Direction == network.DirInbound,
}
// check if observed address seen yet, if so, update it
for _, observedAddr := range oas.addrs[localString] {
if observedAddr.addr.Equal(observed) {
// Don't trump an outbound observation with an inbound
// one.
wasInbound := observedAddr.seenBy[observerString].inbound
isInbound := ob.inbound
ob.inbound = isInbound || wasInbound
if !wasInbound && isInbound {
observedAddr.numInbound++
}
observedAddr.seenBy[observerString] = ob
observedAddr.lastSeen = now
return
}
}
// observed address not seen yet, append it
oa := &observedAddr{
addr: observed,
seenBy: map[string]observation{
observerString: ob,
},
lastSeen: now,
}
if ob.inbound {
oa.numInbound++
}
oas.addrs[localString] = append(oas.addrs[localString], oa)
}
// For a given transport Protocol (TCP/UDP):
//
// 1. If we have an activated address, we are behind an Cone NAT.
// With regards to RFC 3489, this could be either a Full Cone NAT, a Restricted Cone NAT or a
// Port Restricted Cone NAT. However, we do NOT differentiate between them here and simply classify all such NATs as a Cone NAT.
//
// 2. If four different peers observe a different address for us on outbound connections, we
// are MOST probably behind a Symmetric NAT.
//
// Please see the documentation on the enumerations for `network.NATDeviceType` for more details about these NAT Device types
// and how they relate to NAT traversal via Hole Punching.
func (oas *ObservedAddrManager) emitAllNATTypes() {
var allObserved []*observedAddr
for _, addrs := range oas.addrs {
allObserved = append(allObserved, addrs...)
}
hasChanged, natType := oas.emitSpecificNATType(allObserved, ma.P_TCP, network.NATTransportTCP, oas.currentTCPNATDeviceType)
if hasChanged {
oas.currentTCPNATDeviceType = natType
}
hasChanged, natType = oas.emitSpecificNATType(allObserved, ma.P_UDP, network.NATTransportUDP, oas.currentUDPNATDeviceType)
if hasChanged {
oas.currentUDPNATDeviceType = natType
}
}
// returns true along with the new NAT device type if the NAT device type for the given protocol has changed.
// returns false otherwise.
func (oas *ObservedAddrManager) emitSpecificNATType(addrs []*observedAddr, protoCode int, transportProto network.NATTransportProtocol,
currentNATType network.NATDeviceType) (bool, network.NATDeviceType) {
now := time.Now()
seenBy := make(map[string]struct{})
cnt := 0
for _, oa := range addrs {
_, err := oa.addr.ValueForProtocol(protoCode)
if err != nil {
continue
}
// if we have an activated addresses, it's a Cone NAT.
if now.Sub(oa.lastSeen) <= oas.ttl && oa.activated() {
if currentNATType != network.NATDeviceTypeCone {
oas.emitNATDeviceTypeChanged.Emit(event.EvtNATDeviceTypeChanged{
TransportProtocol: transportProto,
NatDeviceType: network.NATDeviceTypeCone,
})
return true, network.NATDeviceTypeCone
}
// our current NAT Device Type is already CONE, nothing to do here.
return false, 0
}
// An observed address on an outbound connection that has ONLY been seen by one peer
if now.Sub(oa.lastSeen) <= oas.ttl && oa.numInbound == 0 && len(oa.seenBy) == 1 {
cnt++
for s := range oa.seenBy {
seenBy[s] = struct{}{}
}
}
}
// If four different peers observe a different address for us on each of four outbound connections, we
// are MOST probably behind a Symmetric NAT.
if cnt >= ActivationThresh && len(seenBy) >= ActivationThresh {
if currentNATType != network.NATDeviceTypeSymmetric {
oas.emitNATDeviceTypeChanged.Emit(event.EvtNATDeviceTypeChanged{
TransportProtocol: transportProto,
NatDeviceType: network.NATDeviceTypeSymmetric,
})
return true, network.NATDeviceTypeSymmetric
}
}
return false, 0
}
func (oas *ObservedAddrManager) Close() error {
oas.closeOnce.Do(func() {
oas.ctxCancel()
oas.mu.Lock()
oas.closed = true
oas.refreshTimer.Stop()
oas.mu.Unlock()
oas.refCount.Wait()
oas.reachabilitySub.Close()
oas.host.Network().StopNotify((*obsAddrNotifiee)(oas))
})
return nil
}
// observerGroup is a function that determines what part of
// a multiaddr counts as a different observer. for example,
// two ipfs nodes at the same IP/TCP transport would get
// the exact same NAT mapping; they would count as the
// same observer. This may protect against NATs who assign
// different ports to addresses at different IP hosts, but
// not TCP ports.
//
// Here, we use the root multiaddr address. This is mostly
// IP addresses. In practice, this is what we want.
func observerGroup(m ma.Multiaddr) string {
// TODO: If IPv6 rolls out we should mark /64 routing zones as one group
first, _ := ma.SplitFirst(m)
return string(first.Bytes())
}
// SetTTL sets the TTL of an observed address manager.
func (oas *ObservedAddrManager) SetTTL(ttl time.Duration) {
oas.mu.Lock()
defer oas.mu.Unlock()
if oas.closed {
return
}
oas.ttl = ttl
// refresh every ttl/2 so we don't forget observations from connected peers
oas.refreshTimer.Reset(ttl / 2)
}
// TTL gets the TTL of an observed address manager.
func (oas *ObservedAddrManager) TTL() time.Duration {
oas.mu.RLock()
defer oas.mu.RUnlock()
return oas.ttl
}
type obsAddrNotifiee ObservedAddrManager
func (on *obsAddrNotifiee) Listen(n network.Network, a ma.Multiaddr) {}
func (on *obsAddrNotifiee) ListenClose(n network.Network, a ma.Multiaddr) {}
func (on *obsAddrNotifiee) Connected(n network.Network, v network.Conn) {}
func (on *obsAddrNotifiee) Disconnected(n network.Network, v network.Conn) {
(*ObservedAddrManager)(on).removeConn(v)
}