mirror of
https://source.quilibrium.com/quilibrium/ceremonyclient.git
synced 2024-11-15 04:35:17 +00:00
556 lines
16 KiB
Go
556 lines
16 KiB
Go
package identify
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"slices"
|
|
"sort"
|
|
"sync"
|
|
|
|
"github.com/libp2p/go-libp2p/core/network"
|
|
|
|
ma "github.com/multiformats/go-multiaddr"
|
|
manet "github.com/multiformats/go-multiaddr/net"
|
|
)
|
|
|
|
// ActivationThresh sets how many times an address must be seen as "activated"
|
|
// and therefore advertised to other peers as an address that the local peer
|
|
// can be contacted on. The "seen" events expire by default after 40 minutes
|
|
// (OwnObservedAddressTTL * ActivationThreshold). The are cleaned up during
|
|
// the GC rounds set by GCInterval.
|
|
var ActivationThresh = 4
|
|
|
|
// observedAddrManagerWorkerChannelSize defines how many addresses can be enqueued
|
|
// for adding to an ObservedAddrManager.
|
|
var observedAddrManagerWorkerChannelSize = 16
|
|
|
|
const maxExternalThinWaistAddrsPerLocalAddr = 3
|
|
|
|
// thinWaist is a struct that stores the address along with it's thin waist prefix and rest of the multiaddr
|
|
type thinWaist struct {
|
|
Addr, TW, Rest ma.Multiaddr
|
|
}
|
|
|
|
// thinWaistWithCount is a thinWaist along with the count of the connection that have it as the local address
|
|
type thinWaistWithCount struct {
|
|
thinWaist
|
|
Count int
|
|
}
|
|
|
|
func thinWaistForm(a ma.Multiaddr) (thinWaist, error) {
|
|
i := 0
|
|
tw, rest, err := ma.SplitFunc(a, func(c ma.Component) bool {
|
|
if i > 1 {
|
|
return true
|
|
}
|
|
switch i {
|
|
case 0:
|
|
if c.Protocol().Code == ma.P_IP4 || c.Protocol().Code == ma.P_IP6 {
|
|
i++
|
|
return false
|
|
}
|
|
return true
|
|
case 1:
|
|
if c.Protocol().Code == ma.P_TCP || c.Protocol().Code == ma.P_UDP {
|
|
i++
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
return false
|
|
})
|
|
if err != nil {
|
|
return thinWaist{}, err
|
|
}
|
|
if i <= 1 {
|
|
return thinWaist{}, fmt.Errorf("not a thinwaist address: %s", a)
|
|
}
|
|
return thinWaist{Addr: a, TW: tw, Rest: rest}, nil
|
|
}
|
|
|
|
// getObserver returns the observer for the multiaddress
|
|
// For an IPv4 multiaddress the observer is the IP address
|
|
// For an IPv6 multiaddress the observer is the first /56 prefix of the IP address
|
|
func getObserver(a ma.Multiaddr) (string, error) {
|
|
ip, err := manet.ToIP(a)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if ip4 := ip.To4(); ip4 != nil {
|
|
return ip4.String(), nil
|
|
}
|
|
// Count /56 prefix as a single observer.
|
|
return ip.Mask(net.CIDRMask(56, 128)).String(), nil
|
|
}
|
|
|
|
// connMultiaddrs provides IsClosed along with network.ConnMultiaddrs. It is easier to mock this than network.Conn
|
|
type connMultiaddrs interface {
|
|
network.ConnMultiaddrs
|
|
IsClosed() bool
|
|
}
|
|
|
|
// observerSetCacheSize is the number of transport sharing the same thinwaist (tcp, ws, wss), (quic, webtransport, webrtc-direct)
|
|
// This is 3 in practice right now, but keep a buffer of 3 extra elements
|
|
const observerSetCacheSize = 5
|
|
|
|
// observerSet is the set of observers who have observed ThinWaistAddr
|
|
type observerSet struct {
|
|
ObservedTWAddr ma.Multiaddr
|
|
ObservedBy map[string]int
|
|
|
|
mu sync.RWMutex // protects following
|
|
cachedMultiaddrs map[string]ma.Multiaddr // cache of localMultiaddr rest(addr - thinwaist) => output multiaddr
|
|
}
|
|
|
|
func (s *observerSet) cacheMultiaddr(addr ma.Multiaddr) ma.Multiaddr {
|
|
if addr == nil {
|
|
return s.ObservedTWAddr
|
|
}
|
|
addrStr := string(addr.Bytes())
|
|
s.mu.RLock()
|
|
res, ok := s.cachedMultiaddrs[addrStr]
|
|
s.mu.RUnlock()
|
|
if ok {
|
|
return res
|
|
}
|
|
|
|
s.mu.Lock()
|
|
// Check if some other go routine added this while we were waiting
|
|
res, ok = s.cachedMultiaddrs[addrStr]
|
|
if ok {
|
|
s.mu.Unlock()
|
|
return res
|
|
}
|
|
if s.cachedMultiaddrs == nil {
|
|
s.cachedMultiaddrs = make(map[string]ma.Multiaddr, observerSetCacheSize)
|
|
}
|
|
if len(s.cachedMultiaddrs) == observerSetCacheSize {
|
|
// remove one entry if we will go over the limit
|
|
for k := range s.cachedMultiaddrs {
|
|
delete(s.cachedMultiaddrs, k)
|
|
break
|
|
}
|
|
}
|
|
s.cachedMultiaddrs[addrStr] = ma.Join(s.ObservedTWAddr, addr)
|
|
mas := s.cachedMultiaddrs[addrStr]
|
|
s.mu.Unlock()
|
|
return mas
|
|
}
|
|
|
|
type observation struct {
|
|
conn connMultiaddrs
|
|
observed ma.Multiaddr
|
|
}
|
|
|
|
// ObservedAddrManager maps connection's local multiaddrs to their externally observable multiaddress
|
|
type ObservedAddrManager struct {
|
|
// Our listen addrs
|
|
listenAddrs func() []ma.Multiaddr
|
|
// Our listen addrs with interface addrs for unspecified addrs
|
|
interfaceListenAddrs func() ([]ma.Multiaddr, error)
|
|
// All host addrs
|
|
hostAddrs func() []ma.Multiaddr
|
|
// Any normalization required before comparing. Useful to remove certhash
|
|
normalize func(ma.Multiaddr) ma.Multiaddr
|
|
// worker channel for new observations
|
|
wch chan observation
|
|
// notified on recording an observation
|
|
addrRecordedNotif chan struct{}
|
|
|
|
// for closing
|
|
wg sync.WaitGroup
|
|
ctx context.Context
|
|
ctxCancel context.CancelFunc
|
|
|
|
mu sync.RWMutex
|
|
// local thin waist => external thin waist => observerSet
|
|
externalAddrs map[string]map[string]*observerSet
|
|
// connObservedTWAddrs maps the connection to the last observed thin waist multiaddr on that connection
|
|
connObservedTWAddrs map[connMultiaddrs]ma.Multiaddr
|
|
// localMultiaddr => thin waist form with the count of the connections the multiaddr
|
|
// was seen on for tracking our local listen addresses
|
|
localAddrs map[string]*thinWaistWithCount
|
|
}
|
|
|
|
// NewObservedAddrManager returns a new address manager using peerstore.OwnObservedAddressTTL as the TTL.
|
|
func NewObservedAddrManager(listenAddrs, hostAddrs func() []ma.Multiaddr,
|
|
interfaceListenAddrs func() ([]ma.Multiaddr, error), normalize func(ma.Multiaddr) ma.Multiaddr) (*ObservedAddrManager, error) {
|
|
if normalize == nil {
|
|
normalize = func(addr ma.Multiaddr) ma.Multiaddr { return addr }
|
|
}
|
|
o := &ObservedAddrManager{
|
|
externalAddrs: make(map[string]map[string]*observerSet),
|
|
connObservedTWAddrs: make(map[connMultiaddrs]ma.Multiaddr),
|
|
localAddrs: make(map[string]*thinWaistWithCount),
|
|
wch: make(chan observation, observedAddrManagerWorkerChannelSize),
|
|
addrRecordedNotif: make(chan struct{}, 1),
|
|
listenAddrs: listenAddrs,
|
|
interfaceListenAddrs: interfaceListenAddrs,
|
|
hostAddrs: hostAddrs,
|
|
normalize: normalize,
|
|
}
|
|
o.ctx, o.ctxCancel = context.WithCancel(context.Background())
|
|
|
|
o.wg.Add(1)
|
|
go o.worker()
|
|
return o, nil
|
|
}
|
|
|
|
// AddrsFor return all activated observed addresses associated with the given
|
|
// (resolved) listen address.
|
|
func (o *ObservedAddrManager) AddrsFor(addr ma.Multiaddr) (addrs []ma.Multiaddr) {
|
|
if addr == nil {
|
|
return nil
|
|
}
|
|
o.mu.RLock()
|
|
tw, err := thinWaistForm(o.normalize(addr))
|
|
if err != nil {
|
|
o.mu.RUnlock()
|
|
return nil
|
|
}
|
|
|
|
observerSets := o.getTopExternalAddrs(string(tw.TW.Bytes()))
|
|
res := make([]ma.Multiaddr, 0, len(observerSets))
|
|
for _, s := range observerSets {
|
|
res = append(res, s.cacheMultiaddr(tw.Rest))
|
|
}
|
|
o.mu.RUnlock()
|
|
return res
|
|
}
|
|
|
|
// Addrs return all activated observed addresses
|
|
func (o *ObservedAddrManager) Addrs() []ma.Multiaddr {
|
|
o.mu.RLock()
|
|
|
|
m := make(map[string][]*observerSet)
|
|
for localTWStr := range o.externalAddrs {
|
|
m[localTWStr] = append(m[localTWStr], o.getTopExternalAddrs(localTWStr)...)
|
|
}
|
|
addrs := make([]ma.Multiaddr, 0, maxExternalThinWaistAddrsPerLocalAddr*5) // assume 5 transports
|
|
for _, t := range o.localAddrs {
|
|
for _, s := range m[string(t.TW.Bytes())] {
|
|
addrs = append(addrs, s.cacheMultiaddr(t.Rest))
|
|
}
|
|
}
|
|
o.mu.RUnlock()
|
|
return addrs
|
|
}
|
|
|
|
func (o *ObservedAddrManager) getTopExternalAddrs(localTWStr string) []*observerSet {
|
|
observerSets := make([]*observerSet, 0, len(o.externalAddrs[localTWStr]))
|
|
for _, v := range o.externalAddrs[localTWStr] {
|
|
if len(v.ObservedBy) >= ActivationThresh {
|
|
observerSets = append(observerSets, v)
|
|
}
|
|
}
|
|
slices.SortFunc(observerSets, func(a, b *observerSet) int {
|
|
diff := len(b.ObservedBy) - len(a.ObservedBy)
|
|
if diff != 0 {
|
|
return diff
|
|
}
|
|
// In case we have elements with equal counts,
|
|
// keep the address list stable by using the lexicographically smaller address
|
|
as := a.ObservedTWAddr.String()
|
|
bs := b.ObservedTWAddr.String()
|
|
if as < bs {
|
|
return -1
|
|
} else if as > bs {
|
|
return 1
|
|
} else {
|
|
return 0
|
|
}
|
|
|
|
})
|
|
n := len(observerSets)
|
|
if n > maxExternalThinWaistAddrsPerLocalAddr {
|
|
n = maxExternalThinWaistAddrsPerLocalAddr
|
|
}
|
|
return observerSets[:n]
|
|
}
|
|
|
|
// Record enqueues an observation for recording
|
|
func (o *ObservedAddrManager) Record(conn connMultiaddrs, observed ma.Multiaddr) {
|
|
select {
|
|
case o.wch <- observation{
|
|
conn: conn,
|
|
observed: observed,
|
|
}:
|
|
default:
|
|
log.Debugw("dropping address observation due to full buffer",
|
|
"from", conn.RemoteMultiaddr(),
|
|
"observed", observed,
|
|
)
|
|
}
|
|
}
|
|
|
|
func (o *ObservedAddrManager) worker() {
|
|
for {
|
|
select {
|
|
case obs := <-o.wch:
|
|
o.maybeRecordObservation(obs.conn, obs.observed)
|
|
case <-o.ctx.Done():
|
|
o.wg.Done()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (o *ObservedAddrManager) shouldRecordObservation(conn connMultiaddrs, observed ma.Multiaddr) (shouldRecord bool, localTW thinWaist, observedTW thinWaist) {
|
|
if conn == nil || observed == nil {
|
|
return false, thinWaist{}, thinWaist{}
|
|
}
|
|
// Ignore observations from loopback nodes. We already know our loopback
|
|
// addresses.
|
|
if manet.IsIPLoopback(observed) {
|
|
return false, thinWaist{}, thinWaist{}
|
|
}
|
|
|
|
// Provided by NAT64 peers, these addresses are specific to the peer and not publicly routable
|
|
if manet.IsNAT64IPv4ConvertedIPv6Addr(observed) {
|
|
return false, thinWaist{}, thinWaist{}
|
|
}
|
|
|
|
// we should only use ObservedAddr when our connection's LocalAddr is one
|
|
// of our ListenAddrs. If we Dial out using an ephemeral addr, knowing that
|
|
// address's external mapping is not very useful because the port will not be
|
|
// the same as the listen addr.
|
|
ifaceaddrs, err := o.interfaceListenAddrs()
|
|
if err != nil {
|
|
log.Infof("failed to get interface listen addrs", err)
|
|
return false, thinWaist{}, thinWaist{}
|
|
}
|
|
|
|
for i, a := range ifaceaddrs {
|
|
ifaceaddrs[i] = o.normalize(a)
|
|
}
|
|
|
|
local := o.normalize(conn.LocalMultiaddr())
|
|
|
|
listenAddrs := o.listenAddrs()
|
|
for i, a := range listenAddrs {
|
|
listenAddrs[i] = o.normalize(a)
|
|
}
|
|
|
|
if !ma.Contains(ifaceaddrs, local) && !ma.Contains(listenAddrs, local) {
|
|
// not in our list
|
|
return false, thinWaist{}, thinWaist{}
|
|
}
|
|
|
|
localTW, err = thinWaistForm(local)
|
|
if err != nil {
|
|
return false, thinWaist{}, thinWaist{}
|
|
}
|
|
observedTW, err = thinWaistForm(o.normalize(observed))
|
|
if err != nil {
|
|
return false, thinWaist{}, thinWaist{}
|
|
}
|
|
|
|
hostAddrs := o.hostAddrs()
|
|
for i, a := range hostAddrs {
|
|
hostAddrs[i] = o.normalize(a)
|
|
}
|
|
|
|
// We should reject the connection if the observation doesn't match the
|
|
// transports of one of our advertised addresses.
|
|
if !HasConsistentTransport(observed, hostAddrs) &&
|
|
!HasConsistentTransport(observed, listenAddrs) {
|
|
log.Debugw(
|
|
"observed multiaddr doesn't match the transports of any announced addresses",
|
|
"from", conn.RemoteMultiaddr(),
|
|
"observed", observed,
|
|
)
|
|
return false, thinWaist{}, thinWaist{}
|
|
}
|
|
|
|
return true, localTW, observedTW
|
|
}
|
|
|
|
func (o *ObservedAddrManager) maybeRecordObservation(conn connMultiaddrs, observed ma.Multiaddr) {
|
|
shouldRecord, localTW, observedTW := o.shouldRecordObservation(conn, observed)
|
|
if !shouldRecord {
|
|
return
|
|
}
|
|
log.Debugw("added own observed listen addr", "observed", observed)
|
|
|
|
o.mu.Lock()
|
|
o.recordObservationUnlocked(conn, localTW, observedTW)
|
|
select {
|
|
case o.addrRecordedNotif <- struct{}{}:
|
|
default:
|
|
}
|
|
o.mu.Unlock()
|
|
}
|
|
|
|
func (o *ObservedAddrManager) recordObservationUnlocked(conn connMultiaddrs, localTW, observedTW thinWaist) {
|
|
if conn.IsClosed() {
|
|
// dont record if the connection is already closed. Any previous observations will be removed in
|
|
// the disconnected callback
|
|
return
|
|
}
|
|
localTWStr := string(localTW.TW.Bytes())
|
|
observedTWStr := string(observedTW.TW.Bytes())
|
|
observer, err := getObserver(conn.RemoteMultiaddr())
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
prevObservedTWAddr, ok := o.connObservedTWAddrs[conn]
|
|
if !ok {
|
|
t, ok := o.localAddrs[string(localTW.Addr.Bytes())]
|
|
if !ok {
|
|
t = &thinWaistWithCount{
|
|
thinWaist: localTW,
|
|
}
|
|
o.localAddrs[string(localTW.Addr.Bytes())] = t
|
|
}
|
|
t.Count++
|
|
} else {
|
|
if prevObservedTWAddr.Equal(observedTW.TW) {
|
|
// we have received the same observation again, nothing to do
|
|
return
|
|
}
|
|
// if we have a previous entry remove it from externalAddrs
|
|
o.removeExternalAddrsUnlocked(observer, localTWStr, string(prevObservedTWAddr.Bytes()))
|
|
// no need to change the localAddrs map here
|
|
}
|
|
o.connObservedTWAddrs[conn] = observedTW.TW
|
|
o.addExternalAddrsUnlocked(observedTW.TW, observer, localTWStr, observedTWStr)
|
|
}
|
|
|
|
func (o *ObservedAddrManager) removeExternalAddrsUnlocked(observer, localTWStr, observedTWStr string) {
|
|
s, ok := o.externalAddrs[localTWStr][observedTWStr]
|
|
if !ok {
|
|
return
|
|
}
|
|
s.ObservedBy[observer]--
|
|
if s.ObservedBy[observer] <= 0 {
|
|
delete(s.ObservedBy, observer)
|
|
}
|
|
if len(s.ObservedBy) == 0 {
|
|
delete(o.externalAddrs[localTWStr], observedTWStr)
|
|
}
|
|
if len(o.externalAddrs[localTWStr]) == 0 {
|
|
delete(o.externalAddrs, localTWStr)
|
|
}
|
|
}
|
|
|
|
func (o *ObservedAddrManager) addExternalAddrsUnlocked(observedTWAddr ma.Multiaddr, observer, localTWStr, observedTWStr string) {
|
|
s, ok := o.externalAddrs[localTWStr][observedTWStr]
|
|
if !ok {
|
|
s = &observerSet{
|
|
ObservedTWAddr: observedTWAddr,
|
|
ObservedBy: make(map[string]int),
|
|
}
|
|
if _, ok := o.externalAddrs[localTWStr]; !ok {
|
|
o.externalAddrs[localTWStr] = make(map[string]*observerSet)
|
|
}
|
|
o.externalAddrs[localTWStr][observedTWStr] = s
|
|
}
|
|
s.ObservedBy[observer]++
|
|
}
|
|
|
|
func (o *ObservedAddrManager) removeConn(conn connMultiaddrs) {
|
|
if conn == nil {
|
|
return
|
|
}
|
|
o.mu.Lock()
|
|
|
|
// normalize before obtaining the thinWaist so that we are always dealing
|
|
// with the normalized form of the address
|
|
localTW, err := thinWaistForm(o.normalize(conn.LocalMultiaddr()))
|
|
if err != nil {
|
|
o.mu.Unlock()
|
|
return
|
|
}
|
|
t, ok := o.localAddrs[string(localTW.Addr.Bytes())]
|
|
if !ok {
|
|
o.mu.Unlock()
|
|
return
|
|
}
|
|
t.Count--
|
|
if t.Count <= 0 {
|
|
delete(o.localAddrs, string(localTW.Addr.Bytes()))
|
|
}
|
|
|
|
observedTWAddr, ok := o.connObservedTWAddrs[conn]
|
|
if !ok {
|
|
o.mu.Unlock()
|
|
return
|
|
}
|
|
delete(o.connObservedTWAddrs, conn)
|
|
observer, err := getObserver(conn.RemoteMultiaddr())
|
|
if err != nil {
|
|
o.mu.Unlock()
|
|
return
|
|
}
|
|
|
|
o.removeExternalAddrsUnlocked(observer, string(localTW.TW.Bytes()), string(observedTWAddr.Bytes()))
|
|
select {
|
|
case o.addrRecordedNotif <- struct{}{}:
|
|
default:
|
|
}
|
|
o.mu.Unlock()
|
|
}
|
|
|
|
func (o *ObservedAddrManager) getNATType() (tcpNATType, udpNATType network.NATDeviceType) {
|
|
o.mu.RLock()
|
|
|
|
var tcpCounts, udpCounts []int
|
|
var tcpTotal, udpTotal int
|
|
for _, m := range o.externalAddrs {
|
|
isTCP := false
|
|
for _, v := range m {
|
|
if _, err := v.ObservedTWAddr.ValueForProtocol(ma.P_TCP); err == nil {
|
|
isTCP = true
|
|
}
|
|
break
|
|
}
|
|
for _, v := range m {
|
|
if isTCP {
|
|
tcpCounts = append(tcpCounts, len(v.ObservedBy))
|
|
tcpTotal += len(v.ObservedBy)
|
|
} else {
|
|
udpCounts = append(udpCounts, len(v.ObservedBy))
|
|
udpTotal += len(v.ObservedBy)
|
|
}
|
|
}
|
|
}
|
|
|
|
sort.Sort(sort.Reverse(sort.IntSlice(tcpCounts)))
|
|
sort.Sort(sort.Reverse(sort.IntSlice(udpCounts)))
|
|
|
|
tcpTopCounts, udpTopCounts := 0, 0
|
|
for i := 0; i < maxExternalThinWaistAddrsPerLocalAddr && i < len(tcpCounts); i++ {
|
|
tcpTopCounts += tcpCounts[i]
|
|
}
|
|
for i := 0; i < maxExternalThinWaistAddrsPerLocalAddr && i < len(udpCounts); i++ {
|
|
udpTopCounts += udpCounts[i]
|
|
}
|
|
|
|
// If the top elements cover more than 1/2 of all the observations, there's a > 50% chance that
|
|
// hole punching based on outputs of observed address manager will succeed
|
|
if tcpTotal >= 3*maxExternalThinWaistAddrsPerLocalAddr {
|
|
if tcpTopCounts >= tcpTotal/2 {
|
|
tcpNATType = network.NATDeviceTypeCone
|
|
} else {
|
|
tcpNATType = network.NATDeviceTypeSymmetric
|
|
}
|
|
}
|
|
if udpTotal >= 3*maxExternalThinWaistAddrsPerLocalAddr {
|
|
if udpTopCounts >= udpTotal/2 {
|
|
udpNATType = network.NATDeviceTypeCone
|
|
} else {
|
|
udpNATType = network.NATDeviceTypeSymmetric
|
|
}
|
|
}
|
|
o.mu.RUnlock()
|
|
return
|
|
}
|
|
|
|
func (o *ObservedAddrManager) Close() error {
|
|
o.ctxCancel()
|
|
o.wg.Wait()
|
|
return nil
|
|
}
|