diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..8a3a276 --- /dev/null +++ b/.envrc @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +use flake + diff --git a/.gitignore b/.gitignore index 99c4ddc..bdb036e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ api/redoc-static.html client2-sim/client2-sim connect/connect.test connect/profile +.direnv diff --git a/client2-sim/go.mod b/client2-sim/go.mod deleted file mode 100644 index d3853d5..0000000 --- a/client2-sim/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module bringyour.com/connect/client2-sim - -go 1.22.0 - -require golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect diff --git a/client2-sim/go.sum b/client2-sim/go.sum deleted file mode 100644 index 3707af3..0000000 --- a/client2-sim/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= -golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= diff --git a/connect/connect.go b/connect/connect.go index cb7d735..be10c56 100644 --- a/connect/connect.go +++ b/connect/connect.go @@ -3,39 +3,37 @@ package connect import ( "errors" // "log" - "fmt" - "encoding/hex" "bytes" + "encoding/hex" + "fmt" "strings" "github.com/oklog/ulid/v2" - "bringyour.com/protocol" + "github.com/bringyour/connect/protocol" ) - const MaxMultihopLength = 8 - // id for message to/from the platform var ControlId = Id{} - // TODO consider having directed transfer paths // TODO SourceTransferPath, DestinationTransferPath // TODO this would avoid the need to check the "masks" // there are three types of transfer paths: -// 1. a full path, which can have either source id and destination id, or stream id -// 2. a source, which can have either source id or stream id. -// This is called the "source mask". -// 3. a destination, which can have either destination id or stream id. -// This is called the "destination mask". +// 1. a full path, which can have either source id and destination id, or stream id +// 2. a source, which can have either source id or stream id. +// This is called the "source mask". +// 3. a destination, which can have either destination id or stream id. +// This is called the "destination mask". +// // comparable type TransferPath struct { - SourceId Id + SourceId Id DestinationId Id - StreamId Id + StreamId Id } func DestinationId(destinationId Id) TransferPath { @@ -141,15 +139,15 @@ func (self TransferPath) SourceMask() TransferPath { func (self TransferPath) DestinationMask() TransferPath { return TransferPath{ DestinationId: self.DestinationId, - StreamId: self.StreamId, + StreamId: self.StreamId, } } func (self TransferPath) Reverse() TransferPath { return TransferPath{ - SourceId: self.DestinationId, + SourceId: self.DestinationId, DestinationId: self.SourceId, - StreamId: self.StreamId, + StreamId: self.StreamId, } } @@ -158,7 +156,7 @@ func (self TransferPath) AddSource(sourceId Id) TransferPath { return self } return TransferPath{ - SourceId: sourceId, + SourceId: sourceId, DestinationId: self.DestinationId, } } @@ -168,7 +166,7 @@ func (self TransferPath) AddDestination(destinationId Id) TransferPath { return self } return TransferPath{ - SourceId: self.SourceId, + SourceId: self.SourceId, DestinationId: destinationId, } } @@ -192,7 +190,6 @@ func (self TransferPath) ToProtobuf() *protocol.TransferPath { return protoTransferPath } - // comparable type Id [16]byte @@ -216,7 +213,7 @@ func RequireIdFromBytes(idBytes []byte) Id { } func ParseId(idStr string) (Id, error) { - return parseUuid(idStr) + return parseUuid(idStr) } func (self Id) Bytes() []byte { @@ -227,7 +224,6 @@ func (self Id) String() string { return encodeUuid(self) } - func (self *Id) MarshalJSON() ([]byte, error) { var buf [16]byte copy(buf[0:16], self[0:16]) @@ -251,7 +247,6 @@ func (self *Id) UnmarshalJSON(src []byte) error { return nil } - func parseUuid(src string) (dst [16]byte, err error) { switch len(src) { case 36: @@ -272,19 +267,17 @@ func parseUuid(src string) (dst [16]byte, err error) { return dst, err } - func encodeUuid(src [16]byte) string { return fmt.Sprintf("%x-%x-%x-%x-%x", src[0:4], src[4:6], src[6:8], src[8:10], src[10:16]) } - // comparable type MultiHopId struct { ids [MaxMultihopLength]Id len int } -func NewMultiHopId(ids ... Id) (MultiHopId, error) { +func NewMultiHopId(ids ...Id) (MultiHopId, error) { if MaxMultihopLength < len(ids) { return MultiHopId{}, fmt.Errorf("Multihop length exceeds maximum: %d < %d", MaxMultihopLength, len(ids)) } @@ -297,7 +290,7 @@ func NewMultiHopId(ids ... Id) (MultiHopId, error) { return multiHopId, nil } -func RequireMultiHopId(ids ... Id) MultiHopId { +func RequireMultiHopId(ids ...Id) MultiHopId { multiHopId, err := NewMultiHopId(ids...) if err != nil { panic(err) @@ -344,7 +337,7 @@ func (self MultiHopId) Tail() Id { if self.len == 0 { panic(errors.New("Cannot call tail on empty multi hop id.")) } - return self.ids[self.len - 1] + return self.ids[self.len-1] } func (self MultiHopId) SplitTail() (MultiHopId, Id) { @@ -357,10 +350,10 @@ func (self MultiHopId) SplitTail() (MultiHopId, Id) { intermediaryIds := MultiHopId{ len: self.len - 1, } - if 0 < self.len - 1 { - copy(intermediaryIds.ids[0:self.len - 1], self.ids[0:self.len - 1]) + if 0 < self.len-1 { + copy(intermediaryIds.ids[0:self.len-1], self.ids[0:self.len-1]) } - return intermediaryIds, self.ids[self.len - 1] + return intermediaryIds, self.ids[self.len-1] } func (self MultiHopId) String() string { @@ -371,8 +364,6 @@ func (self MultiHopId) String() string { return fmt.Sprintf("[%s]", strings.Join(parts, ",")) } - - // use this type when counting bytes type ByteCount = int64 @@ -381,9 +372,9 @@ func kib(c ByteCount) ByteCount { } func mib(c ByteCount) ByteCount { - return c * ByteCount(1024 * 1024) + return c * ByteCount(1024*1024) } func gib(c ByteCount) ByteCount { - return c * ByteCount(1024 * 1024 * 1024) + return c * ByteCount(1024*1024*1024) } diff --git a/connect/frame.go b/connect/frame.go index fff1602..1a45936 100644 --- a/connect/frame.go +++ b/connect/frame.go @@ -1,138 +1,132 @@ package connect import ( - "fmt" - - "google.golang.org/protobuf/proto" + "fmt" - "bringyour.com/protocol" -) + "google.golang.org/protobuf/proto" + "github.com/bringyour/connect/protocol" +) func ToFrame(message proto.Message) (*protocol.Frame, error) { - var messageType protocol.MessageType - switch v := message.(type) { - case *protocol.Pack: - messageType = protocol.MessageType_TransferPack - case *protocol.Ack: - messageType = protocol.MessageType_TransferAck - case *protocol.Contract: - messageType = protocol.MessageType_TransferContract - case *protocol.Provide: - messageType = protocol.MessageType_TransferProvide - case *protocol.Auth: - messageType = protocol.MessageType_TransferAuth - case *protocol.StreamOpen: - messageType = protocol.MessageType_TransferStreamOpen - case *protocol.StreamClose: - messageType = protocol.MessageType_TransferStreamClose - case *protocol.CreateContract: - messageType = protocol.MessageType_TransferCreateContract - case *protocol.CreateContractResult: - messageType = protocol.MessageType_TransferCreateContractResult - case *protocol.CloseContract: - messageType = protocol.MessageType_TransferCloseContract - case *protocol.PeerAudit: - messageType = protocol.MessageType_TransferPeerAudit - case *protocol.SimpleMessage: - messageType = protocol.MessageType_TestSimpleMessage - case *protocol.IpPacketToProvider: - messageType = protocol.MessageType_IpIpPacketToProvider - case *protocol.IpPacketFromProvider: - messageType = protocol.MessageType_IpIpPacketFromProvider - case *protocol.IpPing: - messageType = protocol.MessageType_IpIpPing - default: - return nil, fmt.Errorf("Unknown message type: %T", v) - } - b, err := proto.Marshal(message) - if err != nil { - return nil, err - } - return &protocol.Frame{ - MessageType: messageType, - MessageBytes: b, - }, nil + var messageType protocol.MessageType + switch v := message.(type) { + case *protocol.Pack: + messageType = protocol.MessageType_TransferPack + case *protocol.Ack: + messageType = protocol.MessageType_TransferAck + case *protocol.Contract: + messageType = protocol.MessageType_TransferContract + case *protocol.Provide: + messageType = protocol.MessageType_TransferProvide + case *protocol.Auth: + messageType = protocol.MessageType_TransferAuth + case *protocol.StreamOpen: + messageType = protocol.MessageType_TransferStreamOpen + case *protocol.StreamClose: + messageType = protocol.MessageType_TransferStreamClose + case *protocol.CreateContract: + messageType = protocol.MessageType_TransferCreateContract + case *protocol.CreateContractResult: + messageType = protocol.MessageType_TransferCreateContractResult + case *protocol.CloseContract: + messageType = protocol.MessageType_TransferCloseContract + case *protocol.PeerAudit: + messageType = protocol.MessageType_TransferPeerAudit + case *protocol.SimpleMessage: + messageType = protocol.MessageType_TestSimpleMessage + case *protocol.IpPacketToProvider: + messageType = protocol.MessageType_IpIpPacketToProvider + case *protocol.IpPacketFromProvider: + messageType = protocol.MessageType_IpIpPacketFromProvider + case *protocol.IpPing: + messageType = protocol.MessageType_IpIpPing + default: + return nil, fmt.Errorf("Unknown message type: %T", v) + } + b, err := proto.Marshal(message) + if err != nil { + return nil, err + } + return &protocol.Frame{ + MessageType: messageType, + MessageBytes: b, + }, nil } - func RequireToFrame(message proto.Message) *protocol.Frame { - frame, err := ToFrame(message) - if err != nil { - panic(err) - } - return frame + frame, err := ToFrame(message) + if err != nil { + panic(err) + } + return frame } - func FromFrame(frame *protocol.Frame) (proto.Message, error) { - var message proto.Message - switch frame.MessageType { - case protocol.MessageType_TransferPack: - message = &protocol.Pack{} - case protocol.MessageType_TransferAck: - message = &protocol.Ack{} - case protocol.MessageType_TransferContract: - message = &protocol.Contract{} - case protocol.MessageType_TransferProvide: - message = &protocol.Provide{} - case protocol.MessageType_TransferAuth: - message = &protocol.Auth{} - case protocol.MessageType_TransferStreamOpen: - message = &protocol.StreamOpen{} - case protocol.MessageType_TransferStreamClose: - message = &protocol.StreamClose{} - case protocol.MessageType_TransferCreateContract: - message = &protocol.CreateContract{} - case protocol.MessageType_TransferCreateContractResult: - message = &protocol.CreateContractResult{} - case protocol.MessageType_TransferCloseContract: - message = &protocol.CloseContract{} - case protocol.MessageType_TransferPeerAudit: - message = &protocol.PeerAudit{} - case protocol.MessageType_TestSimpleMessage: - message = &protocol.SimpleMessage{} - case protocol.MessageType_IpIpPacketToProvider: - message = &protocol.IpPacketToProvider{} - case protocol.MessageType_IpIpPacketFromProvider: - message = &protocol.IpPacketFromProvider{} - case protocol.MessageType_IpIpPing: - message = &protocol.IpPing{} - default: - return nil, fmt.Errorf("Unknown message type: %s", frame.MessageType) - } - err := proto.Unmarshal(frame.GetMessageBytes(), message) - if err != nil { - return nil, err - } - return message, nil + var message proto.Message + switch frame.MessageType { + case protocol.MessageType_TransferPack: + message = &protocol.Pack{} + case protocol.MessageType_TransferAck: + message = &protocol.Ack{} + case protocol.MessageType_TransferContract: + message = &protocol.Contract{} + case protocol.MessageType_TransferProvide: + message = &protocol.Provide{} + case protocol.MessageType_TransferAuth: + message = &protocol.Auth{} + case protocol.MessageType_TransferStreamOpen: + message = &protocol.StreamOpen{} + case protocol.MessageType_TransferStreamClose: + message = &protocol.StreamClose{} + case protocol.MessageType_TransferCreateContract: + message = &protocol.CreateContract{} + case protocol.MessageType_TransferCreateContractResult: + message = &protocol.CreateContractResult{} + case protocol.MessageType_TransferCloseContract: + message = &protocol.CloseContract{} + case protocol.MessageType_TransferPeerAudit: + message = &protocol.PeerAudit{} + case protocol.MessageType_TestSimpleMessage: + message = &protocol.SimpleMessage{} + case protocol.MessageType_IpIpPacketToProvider: + message = &protocol.IpPacketToProvider{} + case protocol.MessageType_IpIpPacketFromProvider: + message = &protocol.IpPacketFromProvider{} + case protocol.MessageType_IpIpPing: + message = &protocol.IpPing{} + default: + return nil, fmt.Errorf("Unknown message type: %s", frame.MessageType) + } + err := proto.Unmarshal(frame.GetMessageBytes(), message) + if err != nil { + return nil, err + } + return message, nil } - func RequireFromFrame(frame *protocol.Frame) proto.Message { - message, err := FromFrame(frame) - if err != nil { - panic(err) - } - return message + message, err := FromFrame(frame) + if err != nil { + panic(err) + } + return message } - func EncodeFrame(message proto.Message) ([]byte, error) { - frame, err := ToFrame(message) - if err != nil { - return nil, err - } - b, err := proto.Marshal(frame) - return b, err + frame, err := ToFrame(message) + if err != nil { + return nil, err + } + b, err := proto.Marshal(frame) + return b, err } - func DecodeFrame(b []byte) (proto.Message, error) { - frame := &protocol.Frame{} - err := proto.Unmarshal(b, frame) - if err != nil { - return nil, err - } - return FromFrame(frame) + frame := &protocol.Frame{} + err := proto.Unmarshal(b, frame) + if err != nil { + return nil, err + } + return FromFrame(frame) } diff --git a/connect/go.mod b/connect/go.mod deleted file mode 100644 index fb98214..0000000 --- a/connect/go.mod +++ /dev/null @@ -1,18 +0,0 @@ -module bringyour.com/connect - -go 1.22.0 - -require ( - bringyour.com/protocol v0.0.0 - github.com/go-playground/assert/v2 v2.2.0 - github.com/golang-jwt/jwt/v5 v5.2.0 - github.com/google/gopacket v1.1.19 - github.com/gorilla/websocket v1.5.0 - github.com/oklog/ulid/v2 v2.1.0 - golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 - google.golang.org/protobuf v1.33.0 -) - -require github.com/golang/glog v1.2.1 // indirect - -replace bringyour.com/protocol v0.0.0 => ../protocol/build/bringyour.com/protocol diff --git a/connect/go.sum b/connect/go.sum deleted file mode 100644 index 248d4d9..0000000 --- a/connect/go.sum +++ /dev/null @@ -1,32 +0,0 @@ -github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= -github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= -github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= -github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU= -github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= -github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= diff --git a/connect/ip.go b/connect/ip.go index ae0467e..7109f83 100644 --- a/connect/ip.go +++ b/connect/ip.go @@ -1,36 +1,34 @@ package connect import ( - "net" - "context" - "time" - "sync" - "strconv" - "fmt" - "strings" - "errors" - mathrand "math/rand" - "math" - "io" - "slices" + "context" + "errors" + "fmt" + "io" + "math" + mathrand "math/rand" + "net" + "slices" + "strconv" + "strings" + "sync" + "time" - "github.com/google/gopacket" - "github.com/google/gopacket/layers" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" - "golang.org/x/exp/maps" + "golang.org/x/exp/maps" - // "google.golang.org/protobuf/proto" + // "google.golang.org/protobuf/proto" - "github.com/golang/glog" + "github.com/golang/glog" - "bringyour.com/protocol" + "github.com/bringyour/connect/protocol" ) - // implements user-space NAT (UNAT) and packet inspection // The UNAT emulates a raw socket using user-space sockets. - // use 0 for deadlock testing const DefaultIpBufferSize = 32 @@ -42,151 +40,143 @@ const TcpHeaderSizeWithoutExtensions = 20 const debugVerifyHeaders = false - // send from a raw socket // note `ipProtocol` is not supplied. The implementation must do a packet inspection to determine protocol type SendPacketFunction func(source TransferPath, provideMode protocol.ProvideMode, packet []byte, timeout time.Duration) bool - // receive into a raw socket type ReceivePacketFunction func(source TransferPath, ipProtocol IpProtocol, packet []byte) - type UserNatClient interface { - // `SendPacketFunction` - SendPacket(source TransferPath, provideMode protocol.ProvideMode, packet []byte, timeout time.Duration) bool - Close() - Shuffle() + // `SendPacketFunction` + SendPacket(source TransferPath, provideMode protocol.ProvideMode, packet []byte, timeout time.Duration) bool + Close() + Shuffle() } - func DefaultUdpBufferSettings() *UdpBufferSettings { - return &UdpBufferSettings{ - ReadTimeout: 30 * time.Second, - WriteTimeout: 15 * time.Second, - IdleTimeout: 60 * time.Second, - Mtu: DefaultMtu, - // avoid fragmentation - ReadBufferByteCount: DefaultMtu - max(Ipv4HeaderSizeWithoutExtensions, Ipv6HeaderSize) - max(UdpHeaderSize, TcpHeaderSizeWithoutExtensions), - SequenceBufferSize: DefaultIpBufferSize, - UserLimit: 128, - } + return &UdpBufferSettings{ + ReadTimeout: 30 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 60 * time.Second, + Mtu: DefaultMtu, + // avoid fragmentation + ReadBufferByteCount: DefaultMtu - max(Ipv4HeaderSizeWithoutExtensions, Ipv6HeaderSize) - max(UdpHeaderSize, TcpHeaderSizeWithoutExtensions), + SequenceBufferSize: DefaultIpBufferSize, + UserLimit: 128, + } } - func DefaultTcpBufferSettings() *TcpBufferSettings { - tcpBufferSettings := &TcpBufferSettings{ - ConnectTimeout: 60 * time.Second, - ReadTimeout: 30 * time.Second, - WriteTimeout: 15 * time.Second, - IdleTimeout: 60 * time.Second, - SequenceBufferSize: DefaultIpBufferSize, - Mtu: DefaultMtu, - // avoid fragmentation - ReadBufferByteCount: DefaultMtu - max(Ipv4HeaderSizeWithoutExtensions, Ipv6HeaderSize) - max(UdpHeaderSize, TcpHeaderSizeWithoutExtensions), - WindowSize: int(mib(1)), - UserLimit: 128, - } - return tcpBufferSettings + tcpBufferSettings := &TcpBufferSettings{ + ConnectTimeout: 60 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 60 * time.Second, + SequenceBufferSize: DefaultIpBufferSize, + Mtu: DefaultMtu, + // avoid fragmentation + ReadBufferByteCount: DefaultMtu - max(Ipv4HeaderSizeWithoutExtensions, Ipv6HeaderSize) - max(UdpHeaderSize, TcpHeaderSizeWithoutExtensions), + WindowSize: int(mib(1)), + UserLimit: 128, + } + return tcpBufferSettings } - func DefaultLocalUserNatSettings() *LocalUserNatSettings { - return &LocalUserNatSettings{ - SequenceBufferSize: DefaultIpBufferSize, - BufferTimeout: 5 * time.Second, - UdpBufferSettings: DefaultUdpBufferSettings(), - TcpBufferSettings: DefaultTcpBufferSettings(), - } + return &LocalUserNatSettings{ + SequenceBufferSize: DefaultIpBufferSize, + BufferTimeout: 5 * time.Second, + UdpBufferSettings: DefaultUdpBufferSettings(), + TcpBufferSettings: DefaultTcpBufferSettings(), + } } - type LocalUserNatSettings struct { - SequenceBufferSize int - BufferTimeout time.Duration - UdpBufferSettings *UdpBufferSettings - TcpBufferSettings *TcpBufferSettings + SequenceBufferSize int + BufferTimeout time.Duration + UdpBufferSettings *UdpBufferSettings + TcpBufferSettings *TcpBufferSettings } - // forwards packets using user space sockets // this assumes transfer between the packet source and this is lossless and in order, // so the protocol stack implementations do not implement any retransmit logic type LocalUserNat struct { - ctx context.Context - cancel context.CancelFunc - clientTag string + ctx context.Context + cancel context.CancelFunc + clientTag string - sendPackets chan *SendPacket + sendPackets chan *SendPacket - settings *LocalUserNatSettings + settings *LocalUserNatSettings - // receive callback - receiveCallbacks *CallbackList[ReceivePacketFunction] + // receive callback + receiveCallbacks *CallbackList[ReceivePacketFunction] } func NewLocalUserNatWithDefaults(ctx context.Context, clientTag string) *LocalUserNat { - return NewLocalUserNat(ctx, clientTag, DefaultLocalUserNatSettings()) + return NewLocalUserNat(ctx, clientTag, DefaultLocalUserNatSettings()) } func NewLocalUserNat(ctx context.Context, clientTag string, settings *LocalUserNatSettings) *LocalUserNat { - cancelCtx, cancel := context.WithCancel(ctx) + cancelCtx, cancel := context.WithCancel(ctx) - localUserNat := &LocalUserNat{ - ctx: cancelCtx, - cancel: cancel, - clientTag: clientTag, - sendPackets: make(chan *SendPacket, settings.SequenceBufferSize), - settings: settings, - receiveCallbacks: NewCallbackList[ReceivePacketFunction](), - } - go localUserNat.Run() + localUserNat := &LocalUserNat{ + ctx: cancelCtx, + cancel: cancel, + clientTag: clientTag, + sendPackets: make(chan *SendPacket, settings.SequenceBufferSize), + settings: settings, + receiveCallbacks: NewCallbackList[ReceivePacketFunction](), + } + go localUserNat.Run() - return localUserNat + return localUserNat } // TODO provide mode of the destination determines filtering rules - e.g. local networks // TODO currently filter all local networks and non-encrypted traffic func (self *LocalUserNat) SendPacketWithTimeout(source TransferPath, provideMode protocol.ProvideMode, - packet []byte, timeout time.Duration) bool { - sendPacket := &SendPacket{ - source: source, - provideMode: provideMode, - packet: packet, - } - if timeout < 0 { - select { - case <- self.ctx.Done(): - return false - case self.sendPackets <- sendPacket: - return true - } - } else if 0 == timeout { - select { - case <- self.ctx.Done(): - return false - case self.sendPackets <- sendPacket: - return true - default: - // full - return false - } - } else { - select { - case <- self.ctx.Done(): - return false - case self.sendPackets <- sendPacket: - return true - case <- time.After(timeout): - // full - return false - } - } + packet []byte, timeout time.Duration) bool { + sendPacket := &SendPacket{ + source: source, + provideMode: provideMode, + packet: packet, + } + if timeout < 0 { + select { + case <-self.ctx.Done(): + return false + case self.sendPackets <- sendPacket: + return true + } + } else if 0 == timeout { + select { + case <-self.ctx.Done(): + return false + case self.sendPackets <- sendPacket: + return true + default: + // full + return false + } + } else { + select { + case <-self.ctx.Done(): + return false + case self.sendPackets <- sendPacket: + return true + case <-time.After(timeout): + // full + return false + } + } } // `SendPacketFunction` func (self *LocalUserNat) SendPacket(source TransferPath, provideMode protocol.ProvideMode, packet []byte, timeout time.Duration) bool { - return self.SendPacketWithTimeout(source, provideMode, packet, timeout) + return self.SendPacketWithTimeout(source, provideMode, packet, timeout) } // func (self *LocalUserNat) ReceiveN(source TransferPath, provideMode protocol.ProvideMode, packet []byte, n int) { @@ -194,10 +184,10 @@ func (self *LocalUserNat) SendPacket(source TransferPath, provideMode protocol.P // } func (self *LocalUserNat) AddReceivePacketCallback(receiveCallback ReceivePacketFunction) func() { - callbackId := self.receiveCallbacks.Add(receiveCallback) - return func() { - self.receiveCallbacks.Remove(callbackId) - } + callbackId := self.receiveCallbacks.Add(receiveCallback) + return func() { + self.receiveCallbacks.Remove(callbackId) + } } // func (self *LocalUserNat) RemoveReceivePacketCallback(receiveCallback ReceivePacketFunction) { @@ -206,937 +196,918 @@ func (self *LocalUserNat) AddReceivePacketCallback(receiveCallback ReceivePacket // `ReceivePacketFunction` func (self *LocalUserNat) receive(source TransferPath, ipProtocol IpProtocol, packet []byte) { - for _, receiveCallback := range self.receiveCallbacks.Get() { - HandleError(func() { - receiveCallback(source, ipProtocol, packet) - }) - } + for _, receiveCallback := range self.receiveCallbacks.Get() { + HandleError(func() { + receiveCallback(source, ipProtocol, packet) + }) + } } func (self *LocalUserNat) Run() { - defer self.cancel() - - udp4Buffer := NewUdp4Buffer(self.ctx, self.receive, self.settings.UdpBufferSettings) - udp6Buffer := NewUdp6Buffer(self.ctx, self.receive, self.settings.UdpBufferSettings) - tcp4Buffer := NewTcp4Buffer(self.ctx, self.receive, self.settings.TcpBufferSettings) - tcp6Buffer := NewTcp6Buffer(self.ctx, self.receive, self.settings.TcpBufferSettings) - - for { - select { - case <- self.ctx.Done(): - return - case sendPacket := <- self.sendPackets: - ipPacket := sendPacket.packet - ipVersion := uint8(ipPacket[0]) >> 4 - switch ipVersion { - case 4: - ipv4 := layers.IPv4{} - ipv4.DecodeFromBytes(ipPacket, gopacket.NilDecodeFeedback) - switch ipv4.Protocol { - case layers.IPProtocolUDP: - udp := layers.UDP{} - udp.DecodeFromBytes(ipv4.Payload, gopacket.NilDecodeFeedback) - - c := func()(bool) { - success, err := udp4Buffer.send( - sendPacket.source, - sendPacket.provideMode, - &ipv4, - &udp, - self.settings.BufferTimeout, - ) - return success && err == nil - } - if glog.V(2) { - TraceWithReturn( - fmt.Sprintf("[lnr]send udp4 %s<-%s s(%s)", self.clientTag, sendPacket.source.SourceId, sendPacket.source.StreamId), - c, - ) - } else { - c() - } - case layers.IPProtocolTCP: - tcp := layers.TCP{} - tcp.DecodeFromBytes(ipv4.Payload, gopacket.NilDecodeFeedback) - - c := func()(bool) { - success, err := tcp4Buffer.send( - sendPacket.source, - sendPacket.provideMode, - &ipv4, - &tcp, - self.settings.BufferTimeout, - ) - return success && err == nil - } - if glog.V(2) { - TraceWithReturn( - fmt.Sprintf("[lnr]send tcp4 %s<-%s s(%s)", self.clientTag, sendPacket.source.SourceId, sendPacket.source.StreamId), - c, - ) - } else { - c() - } - default: - // no support for this protocol, drop - } - case 6: - ipv6 := layers.IPv6{} - ipv6.DecodeFromBytes(ipPacket, gopacket.NilDecodeFeedback) - switch ipv6.NextHeader { - case layers.IPProtocolUDP: - udp := layers.UDP{} - udp.DecodeFromBytes(ipv6.Payload, gopacket.NilDecodeFeedback) - - c := func()(bool) { - success, err := udp6Buffer.send( - sendPacket.source, - sendPacket.provideMode, - &ipv6, - &udp, - self.settings.BufferTimeout, - ) - return success && err == nil - } - if glog.V(2) { - TraceWithReturn( - fmt.Sprintf("[lnr]send udp6 %s<-%s s(%s)", self.clientTag, sendPacket.source.SourceId, sendPacket.source.StreamId), - c, - ) - } else { - c() - } - case layers.IPProtocolTCP: - tcp := layers.TCP{} - tcp.DecodeFromBytes(ipv6.Payload, gopacket.NilDecodeFeedback) - - c := func()(bool) { - success, err := tcp6Buffer.send( - sendPacket.source, - sendPacket.provideMode, - &ipv6, - &tcp, - self.settings.BufferTimeout, - ) - return success && err == nil - } - if glog.V(2) { - TraceWithReturn( - fmt.Sprintf("[lnr]send tcp6 %s<-%s s(%s)", self.clientTag, sendPacket.source.SourceId, sendPacket.source.StreamId), - c, - ) - } else { - c() - } - default: - // no support for this protocol, drop - } - } - } - } + defer self.cancel() + + udp4Buffer := NewUdp4Buffer(self.ctx, self.receive, self.settings.UdpBufferSettings) + udp6Buffer := NewUdp6Buffer(self.ctx, self.receive, self.settings.UdpBufferSettings) + tcp4Buffer := NewTcp4Buffer(self.ctx, self.receive, self.settings.TcpBufferSettings) + tcp6Buffer := NewTcp6Buffer(self.ctx, self.receive, self.settings.TcpBufferSettings) + + for { + select { + case <-self.ctx.Done(): + return + case sendPacket := <-self.sendPackets: + ipPacket := sendPacket.packet + ipVersion := uint8(ipPacket[0]) >> 4 + switch ipVersion { + case 4: + ipv4 := layers.IPv4{} + ipv4.DecodeFromBytes(ipPacket, gopacket.NilDecodeFeedback) + switch ipv4.Protocol { + case layers.IPProtocolUDP: + udp := layers.UDP{} + udp.DecodeFromBytes(ipv4.Payload, gopacket.NilDecodeFeedback) + + c := func() bool { + success, err := udp4Buffer.send( + sendPacket.source, + sendPacket.provideMode, + &ipv4, + &udp, + self.settings.BufferTimeout, + ) + return success && err == nil + } + if glog.V(2) { + TraceWithReturn( + fmt.Sprintf("[lnr]send udp4 %s<-%s s(%s)", self.clientTag, sendPacket.source.SourceId, sendPacket.source.StreamId), + c, + ) + } else { + c() + } + case layers.IPProtocolTCP: + tcp := layers.TCP{} + tcp.DecodeFromBytes(ipv4.Payload, gopacket.NilDecodeFeedback) + + c := func() bool { + success, err := tcp4Buffer.send( + sendPacket.source, + sendPacket.provideMode, + &ipv4, + &tcp, + self.settings.BufferTimeout, + ) + return success && err == nil + } + if glog.V(2) { + TraceWithReturn( + fmt.Sprintf("[lnr]send tcp4 %s<-%s s(%s)", self.clientTag, sendPacket.source.SourceId, sendPacket.source.StreamId), + c, + ) + } else { + c() + } + default: + // no support for this protocol, drop + } + case 6: + ipv6 := layers.IPv6{} + ipv6.DecodeFromBytes(ipPacket, gopacket.NilDecodeFeedback) + switch ipv6.NextHeader { + case layers.IPProtocolUDP: + udp := layers.UDP{} + udp.DecodeFromBytes(ipv6.Payload, gopacket.NilDecodeFeedback) + + c := func() bool { + success, err := udp6Buffer.send( + sendPacket.source, + sendPacket.provideMode, + &ipv6, + &udp, + self.settings.BufferTimeout, + ) + return success && err == nil + } + if glog.V(2) { + TraceWithReturn( + fmt.Sprintf("[lnr]send udp6 %s<-%s s(%s)", self.clientTag, sendPacket.source.SourceId, sendPacket.source.StreamId), + c, + ) + } else { + c() + } + case layers.IPProtocolTCP: + tcp := layers.TCP{} + tcp.DecodeFromBytes(ipv6.Payload, gopacket.NilDecodeFeedback) + + c := func() bool { + success, err := tcp6Buffer.send( + sendPacket.source, + sendPacket.provideMode, + &ipv6, + &tcp, + self.settings.BufferTimeout, + ) + return success && err == nil + } + if glog.V(2) { + TraceWithReturn( + fmt.Sprintf("[lnr]send tcp6 %s<-%s s(%s)", self.clientTag, sendPacket.source.SourceId, sendPacket.source.StreamId), + c, + ) + } else { + c() + } + default: + // no support for this protocol, drop + } + } + } + } } func (self *LocalUserNat) Close() { - self.cancel() + self.cancel() } type SendPacket struct { - source TransferPath - provideMode protocol.ProvideMode - packet []byte + source TransferPath + provideMode protocol.ProvideMode + packet []byte } - // comparable type BufferId4 struct { - source TransferPath - sourceIp [4]byte - sourcePort int - destinationIp [4]byte - destinationPort int + source TransferPath + sourceIp [4]byte + sourcePort int + destinationIp [4]byte + destinationPort int } func NewBufferId4(source TransferPath, sourceIp net.IP, sourcePort int, destinationIp net.IP, destinationPort int) BufferId4 { - return BufferId4{ - source: source, - sourceIp: [4]byte(sourceIp), - sourcePort: sourcePort, - destinationIp: [4]byte(destinationIp), - destinationPort: destinationPort, - } + return BufferId4{ + source: source, + sourceIp: [4]byte(sourceIp), + sourcePort: sourcePort, + destinationIp: [4]byte(destinationIp), + destinationPort: destinationPort, + } } - // comparable type BufferId6 struct { - source TransferPath - sourceIp [16]byte - sourcePort int - destinationIp [16]byte - destinationPort int + source TransferPath + sourceIp [16]byte + sourcePort int + destinationIp [16]byte + destinationPort int } func NewBufferId6(source TransferPath, sourceIp net.IP, sourcePort int, destinationIp net.IP, destinationPort int) BufferId6 { - return BufferId6{ - source: source, - sourceIp: [16]byte(sourceIp), - sourcePort: sourcePort, - destinationIp: [16]byte(destinationIp), - destinationPort: destinationPort, - } + return BufferId6{ + source: source, + sourceIp: [16]byte(sourceIp), + sourcePort: sourcePort, + destinationIp: [16]byte(destinationIp), + destinationPort: destinationPort, + } } - type UdpBufferSettings struct { - ReadTimeout time.Duration - WriteTimeout time.Duration - IdleTimeout time.Duration - Mtu int - ReadBufferByteCount int - SequenceBufferSize int - // the number of open sockets per user - // uses an lru cleanup where new sockets over the limit close old sockets - UserLimit int + ReadTimeout time.Duration + WriteTimeout time.Duration + IdleTimeout time.Duration + Mtu int + ReadBufferByteCount int + SequenceBufferSize int + // the number of open sockets per user + // uses an lru cleanup where new sockets over the limit close old sockets + UserLimit int } - type Udp4Buffer struct { - UdpBuffer[BufferId4] + UdpBuffer[BufferId4] } func NewUdp4Buffer(ctx context.Context, receiveCallback ReceivePacketFunction, - udpBufferSettings *UdpBufferSettings) *Udp4Buffer { - return &Udp4Buffer{ - UdpBuffer: *newUdpBuffer[BufferId4](ctx, receiveCallback, udpBufferSettings), - } + udpBufferSettings *UdpBufferSettings) *Udp4Buffer { + return &Udp4Buffer{ + UdpBuffer: *newUdpBuffer[BufferId4](ctx, receiveCallback, udpBufferSettings), + } } func (self *Udp4Buffer) send(source TransferPath, provideMode protocol.ProvideMode, - ipv4 *layers.IPv4, udp *layers.UDP, timeout time.Duration) (bool, error) { - bufferId := NewBufferId4( - source, - ipv4.SrcIP, int(udp.SrcPort), - ipv4.DstIP, int(udp.DstPort), - ) - - return self.udpSend( - bufferId, - ipv4.SrcIP, - ipv4.DstIP, - source, - provideMode, - 4, - udp, - timeout, - ) + ipv4 *layers.IPv4, udp *layers.UDP, timeout time.Duration) (bool, error) { + bufferId := NewBufferId4( + source, + ipv4.SrcIP, int(udp.SrcPort), + ipv4.DstIP, int(udp.DstPort), + ) + + return self.udpSend( + bufferId, + ipv4.SrcIP, + ipv4.DstIP, + source, + provideMode, + 4, + udp, + timeout, + ) } - type Udp6Buffer struct { - UdpBuffer[BufferId6] + UdpBuffer[BufferId6] } func NewUdp6Buffer(ctx context.Context, receiveCallback ReceivePacketFunction, - udpBufferSettings *UdpBufferSettings) *Udp6Buffer { - return &Udp6Buffer{ - UdpBuffer: *newUdpBuffer[BufferId6](ctx, receiveCallback, udpBufferSettings), - } + udpBufferSettings *UdpBufferSettings) *Udp6Buffer { + return &Udp6Buffer{ + UdpBuffer: *newUdpBuffer[BufferId6](ctx, receiveCallback, udpBufferSettings), + } } func (self *Udp6Buffer) send(source TransferPath, provideMode protocol.ProvideMode, - ipv6 *layers.IPv6, udp *layers.UDP, timeout time.Duration) (bool, error) { - bufferId := NewBufferId6( - source, - ipv6.SrcIP, int(udp.SrcPort), - ipv6.DstIP, int(udp.DstPort), - ) - - return self.udpSend( - bufferId, - ipv6.SrcIP, - ipv6.DstIP, - source, - provideMode, - 6, - udp, - timeout, - ) + ipv6 *layers.IPv6, udp *layers.UDP, timeout time.Duration) (bool, error) { + bufferId := NewBufferId6( + source, + ipv6.SrcIP, int(udp.SrcPort), + ipv6.DstIP, int(udp.DstPort), + ) + + return self.udpSend( + bufferId, + ipv6.SrcIP, + ipv6.DstIP, + source, + provideMode, + 6, + udp, + timeout, + ) } - type UdpBuffer[BufferId comparable] struct { - ctx context.Context - receiveCallback ReceivePacketFunction - udpBufferSettings *UdpBufferSettings + ctx context.Context + receiveCallback ReceivePacketFunction + udpBufferSettings *UdpBufferSettings - mutex sync.Mutex + mutex sync.Mutex - sequences map[BufferId]*UdpSequence - sourceSequences map[TransferPath]map[BufferId]*UdpSequence + sequences map[BufferId]*UdpSequence + sourceSequences map[TransferPath]map[BufferId]*UdpSequence } func newUdpBuffer[BufferId comparable]( - ctx context.Context, - receiveCallback ReceivePacketFunction, - udpBufferSettings *UdpBufferSettings, + ctx context.Context, + receiveCallback ReceivePacketFunction, + udpBufferSettings *UdpBufferSettings, ) *UdpBuffer[BufferId] { - return &UdpBuffer[BufferId]{ - ctx: ctx, - receiveCallback: receiveCallback, - udpBufferSettings: udpBufferSettings, - sequences: map[BufferId]*UdpSequence{}, - sourceSequences: map[TransferPath]map[BufferId]*UdpSequence{}, - } + return &UdpBuffer[BufferId]{ + ctx: ctx, + receiveCallback: receiveCallback, + udpBufferSettings: udpBufferSettings, + sequences: map[BufferId]*UdpSequence{}, + sourceSequences: map[TransferPath]map[BufferId]*UdpSequence{}, + } } - func (self *UdpBuffer[BufferId]) udpSend( - bufferId BufferId, - sourceIp net.IP, - destinationIp net.IP, - source TransferPath, - provideMode protocol.ProvideMode, - ipVersion int, - udp *layers.UDP, - timeout time.Duration, + bufferId BufferId, + sourceIp net.IP, + destinationIp net.IP, + source TransferPath, + provideMode protocol.ProvideMode, + ipVersion int, + udp *layers.UDP, + timeout time.Duration, ) (bool, error) { - initSequence := func(skip *UdpSequence)(*UdpSequence) { - self.mutex.Lock() - defer self.mutex.Unlock() - - sequence, ok := self.sequences[bufferId] - if ok { - if skip == nil || skip != sequence { - return sequence - } else { - sequence.Cancel() - delete(self.sequences, bufferId) - sourceSequences := self.sourceSequences[sequence.source] - delete(sourceSequences, bufferId) - if 0 == len(sourceSequences) { - delete(self.sourceSequences, sequence.source) - } - } - } - - if 0 < self.udpBufferSettings.UserLimit { - // limit the total connections per source to avoid blowing up the ulimit - if sourceSequences := self.sourceSequences[source]; self.udpBufferSettings.UserLimit < len(sourceSequences) { - applyLruUserLimit(maps.Values(sourceSequences), self.udpBufferSettings.UserLimit, func(sequence *UdpSequence)(bool) { - glog.Infof( - "[lnr]udp limit source %s->%s\n", - source, - net.JoinHostPort( - sequence.destinationIp.String(), - strconv.Itoa(int(sequence.destinationPort)), - ), - ) - return true - }) - } - } - - sequence = NewUdpSequence( - self.ctx, - self.receiveCallback, - source, - ipVersion, - sourceIp, - udp.SrcPort, - destinationIp, - udp.DstPort, - self.udpBufferSettings, - ) - self.sequences[bufferId] = sequence - go func() { - sequence.Run() - - self.mutex.Lock() - defer self.mutex.Unlock() - sequence.Close() - // clean up - if sequence == self.sequences[bufferId] { - delete(self.sequences, bufferId) - sourceSequences := self.sourceSequences[sequence.source] - delete(sourceSequences, bufferId) - if 0 == len(sourceSequences) { - delete(self.sourceSequences, sequence.source) - } - } - }() - return sequence - } - - sendItem := &UdpSendItem{ - provideMode: provideMode, - udp: udp, - } - sequence := initSequence(nil) - if success, err := sequence.send(sendItem, timeout); err == nil { - return success, nil - } else { - // sequence closed - return initSequence(sequence).send(sendItem, timeout) - } + initSequence := func(skip *UdpSequence) *UdpSequence { + self.mutex.Lock() + defer self.mutex.Unlock() + + sequence, ok := self.sequences[bufferId] + if ok { + if skip == nil || skip != sequence { + return sequence + } else { + sequence.Cancel() + delete(self.sequences, bufferId) + sourceSequences := self.sourceSequences[sequence.source] + delete(sourceSequences, bufferId) + if 0 == len(sourceSequences) { + delete(self.sourceSequences, sequence.source) + } + } + } + + if 0 < self.udpBufferSettings.UserLimit { + // limit the total connections per source to avoid blowing up the ulimit + if sourceSequences := self.sourceSequences[source]; self.udpBufferSettings.UserLimit < len(sourceSequences) { + applyLruUserLimit(maps.Values(sourceSequences), self.udpBufferSettings.UserLimit, func(sequence *UdpSequence) bool { + glog.Infof( + "[lnr]udp limit source %s->%s\n", + source, + net.JoinHostPort( + sequence.destinationIp.String(), + strconv.Itoa(int(sequence.destinationPort)), + ), + ) + return true + }) + } + } + + sequence = NewUdpSequence( + self.ctx, + self.receiveCallback, + source, + ipVersion, + sourceIp, + udp.SrcPort, + destinationIp, + udp.DstPort, + self.udpBufferSettings, + ) + self.sequences[bufferId] = sequence + go func() { + sequence.Run() + + self.mutex.Lock() + defer self.mutex.Unlock() + sequence.Close() + // clean up + if sequence == self.sequences[bufferId] { + delete(self.sequences, bufferId) + sourceSequences := self.sourceSequences[sequence.source] + delete(sourceSequences, bufferId) + if 0 == len(sourceSequences) { + delete(self.sourceSequences, sequence.source) + } + } + }() + return sequence + } + + sendItem := &UdpSendItem{ + provideMode: provideMode, + udp: udp, + } + sequence := initSequence(nil) + if success, err := sequence.send(sendItem, timeout); err == nil { + return success, nil + } else { + // sequence closed + return initSequence(sequence).send(sendItem, timeout) + } } - - type UdpSequence struct { - ctx context.Context - cancel context.CancelFunc - receiveCallback ReceivePacketFunction - udpBufferSettings *UdpBufferSettings + ctx context.Context + cancel context.CancelFunc + receiveCallback ReceivePacketFunction + udpBufferSettings *UdpBufferSettings - sendItems chan *UdpSendItem + sendItems chan *UdpSendItem - idleCondition *IdleCondition + idleCondition *IdleCondition - StreamState + StreamState } func NewUdpSequence(ctx context.Context, receiveCallback ReceivePacketFunction, - source TransferPath, - ipVersion int, - sourceIp net.IP, sourcePort layers.UDPPort, - destinationIp net.IP, destinationPort layers.UDPPort, - udpBufferSettings *UdpBufferSettings) *UdpSequence { - cancelCtx, cancel := context.WithCancel(ctx) - streamState := StreamState{ - source: source, - ipVersion: ipVersion, - sourceIp: sourceIp, - sourcePort: sourcePort, - destinationIp: destinationIp, - destinationPort: destinationPort, - userLimited: *newUserLimited(), - } - return &UdpSequence{ - ctx: cancelCtx, - cancel: cancel, - receiveCallback: receiveCallback, - sendItems: make(chan *UdpSendItem, udpBufferSettings.SequenceBufferSize), - udpBufferSettings: udpBufferSettings, - idleCondition: NewIdleCondition(), - StreamState: streamState, - } + source TransferPath, + ipVersion int, + sourceIp net.IP, sourcePort layers.UDPPort, + destinationIp net.IP, destinationPort layers.UDPPort, + udpBufferSettings *UdpBufferSettings) *UdpSequence { + cancelCtx, cancel := context.WithCancel(ctx) + streamState := StreamState{ + source: source, + ipVersion: ipVersion, + sourceIp: sourceIp, + sourcePort: sourcePort, + destinationIp: destinationIp, + destinationPort: destinationPort, + userLimited: *newUserLimited(), + } + return &UdpSequence{ + ctx: cancelCtx, + cancel: cancel, + receiveCallback: receiveCallback, + sendItems: make(chan *UdpSendItem, udpBufferSettings.SequenceBufferSize), + udpBufferSettings: udpBufferSettings, + idleCondition: NewIdleCondition(), + StreamState: streamState, + } } func (self *UdpSequence) send(sendItem *UdpSendItem, timeout time.Duration) (bool, error) { - if !self.idleCondition.UpdateOpen() { - return false, nil - } - defer self.idleCondition.UpdateClose() - - select { - case <- self.ctx.Done(): - return false, errors.New("Done.") - default: - } - - if timeout < 0 { - select { - case <- self.ctx.Done(): - return false, errors.New("Done.") - case self.sendItems <- sendItem: - return true, nil - } - } else if timeout == 0 { - select { - case <- self.ctx.Done(): - return false, errors.New("Done.") - case self.sendItems <- sendItem: - return true, nil - default: - return false, nil - } - } else { - select { - case <- self.ctx.Done(): - return false, errors.New("Done.") - case self.sendItems <- sendItem: - return true, nil - case <- time.After(timeout): - return false, nil - } - } + if !self.idleCondition.UpdateOpen() { + return false, nil + } + defer self.idleCondition.UpdateClose() + + select { + case <-self.ctx.Done(): + return false, errors.New("Done.") + default: + } + + if timeout < 0 { + select { + case <-self.ctx.Done(): + return false, errors.New("Done.") + case self.sendItems <- sendItem: + return true, nil + } + } else if timeout == 0 { + select { + case <-self.ctx.Done(): + return false, errors.New("Done.") + case self.sendItems <- sendItem: + return true, nil + default: + return false, nil + } + } else { + select { + case <-self.ctx.Done(): + return false, errors.New("Done.") + case self.sendItems <- sendItem: + return true, nil + case <-time.After(timeout): + return false, nil + } + } } func (self *UdpSequence) Run() { - defer self.cancel() - - receive := func(packet []byte) { - self.receiveCallback(self.source, IpProtocolUdp, packet) - } - - glog.V(2).Infof("[init]udp connect\n") - socket, err := net.Dial( - "udp", - self.DestinationAuthority(), - ) - if err != nil { - glog.Infof("[init]udp connect error = %s\n", err) - return - } - defer socket.Close() - self.UpdateLastActivityTime() - glog.V(2).Infof("[init]connect success\n") - - go func() { - defer self.cancel() - - buffer := make([]byte, self.udpBufferSettings.ReadBufferByteCount) - - for forwardIter := uint64(0); ; forwardIter += 1 { - select { - case <- self.ctx.Done(): - return - default: - } - - - readTimeout := time.Now().Add(self.udpBufferSettings.ReadTimeout) - socket.SetReadDeadline(readTimeout) - n, err := socket.Read(buffer) - - if err != nil { - glog.Infof("[f%d]udp receive err = %s\n", forwardIter, err) - } - - if 0 < n { - self.UpdateLastActivityTime() - - packets, packetsErr := self.DataPackets(buffer, n, self.udpBufferSettings.Mtu) - if packetsErr != nil { - glog.Infof("[f%d]udp receive packets error = %s\n", forwardIter, packetsErr) - return - } - if 1 < len(packets) { - glog.V(2).Infof("[f%d]udp receive segemented packets = %d\n", forwardIter, len(packets)) - } - for _, packet := range packets { - glog.V(1).Infof("[f%d]udp receive %d\n", forwardIter, len(packet)) - receive(packet) - } - } - - if err != nil { - if err == io.EOF { - return - } else if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - glog.Infof("[f%d]timeout\n", forwardIter) - return - } else { - // some other error - return - } - } - } - }() - - go func() { - defer self.cancel() - - for sendIter := 0; ; sendIter += 1 { - checkpointId := self.idleCondition.Checkpoint() - select { - case <- self.ctx.Done(): - return - case sendItem := <- self.sendItems: - writeEndTime := time.Now().Add(self.udpBufferSettings.WriteTimeout) - - payload := sendItem.udp.Payload - - for i := 0; i < len(payload); { - select { - case <- self.ctx.Done(): - return - default: - } - - socket.SetWriteDeadline(writeEndTime) - n, err := socket.Write(payload[i:]) - - if err == nil { - glog.V(2).Infof("[f%d]udp forward %d\n", sendIter, n) - } else { - glog.Infof("[f%d]udp forward %d error = %s", sendIter, n, err) - } - - if 0 < n { - self.UpdateLastActivityTime() - - j := i - i += n - glog.V(2).Infof("[f%d]udp forward %d/%d -> %d/%d +%d\n", sendIter, j, len(payload), i, len(payload), n) - } - - - if err != nil { - if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - return - } else { - // some other error - return - } - } - } - case <- time.After(self.udpBufferSettings.IdleTimeout): - if self.idleCondition.Close(checkpointId) { - // close the sequence - return - } - // else there pending updates - } - } - }() - - select { - case <- self.ctx.Done(): - } + defer self.cancel() + + receive := func(packet []byte) { + self.receiveCallback(self.source, IpProtocolUdp, packet) + } + + glog.V(2).Infof("[init]udp connect\n") + socket, err := net.Dial( + "udp", + self.DestinationAuthority(), + ) + if err != nil { + glog.Infof("[init]udp connect error = %s\n", err) + return + } + defer socket.Close() + self.UpdateLastActivityTime() + glog.V(2).Infof("[init]connect success\n") + + go func() { + defer self.cancel() + + buffer := make([]byte, self.udpBufferSettings.ReadBufferByteCount) + + for forwardIter := uint64(0); ; forwardIter += 1 { + select { + case <-self.ctx.Done(): + return + default: + } + + readTimeout := time.Now().Add(self.udpBufferSettings.ReadTimeout) + socket.SetReadDeadline(readTimeout) + n, err := socket.Read(buffer) + + if err != nil { + glog.Infof("[f%d]udp receive err = %s\n", forwardIter, err) + } + + if 0 < n { + self.UpdateLastActivityTime() + + packets, packetsErr := self.DataPackets(buffer, n, self.udpBufferSettings.Mtu) + if packetsErr != nil { + glog.Infof("[f%d]udp receive packets error = %s\n", forwardIter, packetsErr) + return + } + if 1 < len(packets) { + glog.V(2).Infof("[f%d]udp receive segemented packets = %d\n", forwardIter, len(packets)) + } + for _, packet := range packets { + glog.V(1).Infof("[f%d]udp receive %d\n", forwardIter, len(packet)) + receive(packet) + } + } + + if err != nil { + if err == io.EOF { + return + } else if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + glog.Infof("[f%d]timeout\n", forwardIter) + return + } else { + // some other error + return + } + } + } + }() + + go func() { + defer self.cancel() + + for sendIter := 0; ; sendIter += 1 { + checkpointId := self.idleCondition.Checkpoint() + select { + case <-self.ctx.Done(): + return + case sendItem := <-self.sendItems: + writeEndTime := time.Now().Add(self.udpBufferSettings.WriteTimeout) + + payload := sendItem.udp.Payload + + for i := 0; i < len(payload); { + select { + case <-self.ctx.Done(): + return + default: + } + + socket.SetWriteDeadline(writeEndTime) + n, err := socket.Write(payload[i:]) + + if err == nil { + glog.V(2).Infof("[f%d]udp forward %d\n", sendIter, n) + } else { + glog.Infof("[f%d]udp forward %d error = %s", sendIter, n, err) + } + + if 0 < n { + self.UpdateLastActivityTime() + + j := i + i += n + glog.V(2).Infof("[f%d]udp forward %d/%d -> %d/%d +%d\n", sendIter, j, len(payload), i, len(payload), n) + } + + if err != nil { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + return + } else { + // some other error + return + } + } + } + case <-time.After(self.udpBufferSettings.IdleTimeout): + if self.idleCondition.Close(checkpointId) { + // close the sequence + return + } + // else there pending updates + } + } + }() + + select { + case <-self.ctx.Done(): + } } func (self *UdpSequence) Cancel() { - self.cancel() + self.cancel() } func (self *UdpSequence) Close() { - self.cancel() + self.cancel() } - type UdpSendItem struct { - source TransferPath - provideMode protocol.ProvideMode - udp *layers.UDP + source TransferPath + provideMode protocol.ProvideMode + udp *layers.UDP } type StreamState struct { - source TransferPath - ipVersion int - sourceIp net.IP - sourcePort layers.UDPPort - destinationIp net.IP - destinationPort layers.UDPPort + source TransferPath + ipVersion int + sourceIp net.IP + sourcePort layers.UDPPort + destinationIp net.IP + destinationPort layers.UDPPort - userLimited + userLimited } func (self *StreamState) SourceAuthority() string { - return net.JoinHostPort( - self.sourceIp.String(), - strconv.Itoa(int(self.sourcePort)), - ) + return net.JoinHostPort( + self.sourceIp.String(), + strconv.Itoa(int(self.sourcePort)), + ) } func (self *StreamState) DestinationAuthority() string { - return net.JoinHostPort( - self.destinationIp.String(), - strconv.Itoa(int(self.destinationPort)), - ) + return net.JoinHostPort( + self.destinationIp.String(), + strconv.Itoa(int(self.destinationPort)), + ) } func (self *StreamState) DataPackets(payload []byte, n int, mtu int) ([][]byte, error) { - headerSize := 0 - var ip gopacket.NetworkLayer - switch self.ipVersion { - case 4: - ip = &layers.IPv4{ - Version: 4, - TTL: 64, - SrcIP: self.destinationIp, - DstIP: self.sourceIp, - Protocol: layers.IPProtocolUDP, - } - headerSize += Ipv4HeaderSizeWithoutExtensions - case 6: - ip = &layers.IPv6{ - Version: 6, - HopLimit: 64, - SrcIP: self.destinationIp, - DstIP: self.sourceIp, - NextHeader: layers.IPProtocolUDP, - } - headerSize += Ipv6HeaderSize - } - - udp := layers.UDP{ - SrcPort: self.destinationPort, - DstPort: self.sourcePort, - } - udp.SetNetworkLayerForChecksum(ip) - headerSize += UdpHeaderSize - - options := gopacket.SerializeOptions{ - ComputeChecksums: true, - FixLengths: true, - } - - - if debugVerifyHeaders { - buffer := gopacket.NewSerializeBufferExpectedSize(headerSize, 0) - err := gopacket.SerializeLayers(buffer, options, - ip.(gopacket.SerializableLayer), - &udp, - ) - if err != nil { - return nil, err - } - packetHeaders := buffer.Bytes() - if headerSize != len(packetHeaders) { - return nil, errors.New(fmt.Sprintf("Header check failed %d <> %d", headerSize, len(packetHeaders))) - } - } - - - if headerSize + n <= mtu { - buffer := gopacket.NewSerializeBufferExpectedSize(headerSize + n, 0) - err := gopacket.SerializeLayers(buffer, options, - ip.(gopacket.SerializableLayer), - &udp, - gopacket.Payload(payload[0:n]), - ) - if err != nil { - return nil, err - } - packet := buffer.Bytes() - return [][]byte{packet}, nil - } else { - // fragment - buffer := gopacket.NewSerializeBufferExpectedSize(mtu, 0) - packetSize := mtu - headerSize - packets := make([][]byte, 0, (n + packetSize) / packetSize) - for i := 0; i < n; { - j := min(i + packetSize, n) - err := gopacket.SerializeLayers(buffer, options, - ip.(gopacket.SerializableLayer), - &udp, - gopacket.Payload(payload[i:j]), - ) - if err != nil { - return nil, err - } - packet := buffer.Bytes() - packetCopy := make([]byte, len(packet)) - copy(packetCopy, packet) - packets = append(packets, packetCopy) - buffer.Clear() - i = j - } - return packets, nil - } + headerSize := 0 + var ip gopacket.NetworkLayer + switch self.ipVersion { + case 4: + ip = &layers.IPv4{ + Version: 4, + TTL: 64, + SrcIP: self.destinationIp, + DstIP: self.sourceIp, + Protocol: layers.IPProtocolUDP, + } + headerSize += Ipv4HeaderSizeWithoutExtensions + case 6: + ip = &layers.IPv6{ + Version: 6, + HopLimit: 64, + SrcIP: self.destinationIp, + DstIP: self.sourceIp, + NextHeader: layers.IPProtocolUDP, + } + headerSize += Ipv6HeaderSize + } + + udp := layers.UDP{ + SrcPort: self.destinationPort, + DstPort: self.sourcePort, + } + udp.SetNetworkLayerForChecksum(ip) + headerSize += UdpHeaderSize + + options := gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + } + + if debugVerifyHeaders { + buffer := gopacket.NewSerializeBufferExpectedSize(headerSize, 0) + err := gopacket.SerializeLayers(buffer, options, + ip.(gopacket.SerializableLayer), + &udp, + ) + if err != nil { + return nil, err + } + packetHeaders := buffer.Bytes() + if headerSize != len(packetHeaders) { + return nil, errors.New(fmt.Sprintf("Header check failed %d <> %d", headerSize, len(packetHeaders))) + } + } + + if headerSize+n <= mtu { + buffer := gopacket.NewSerializeBufferExpectedSize(headerSize+n, 0) + err := gopacket.SerializeLayers(buffer, options, + ip.(gopacket.SerializableLayer), + &udp, + gopacket.Payload(payload[0:n]), + ) + if err != nil { + return nil, err + } + packet := buffer.Bytes() + return [][]byte{packet}, nil + } else { + // fragment + buffer := gopacket.NewSerializeBufferExpectedSize(mtu, 0) + packetSize := mtu - headerSize + packets := make([][]byte, 0, (n+packetSize)/packetSize) + for i := 0; i < n; { + j := min(i+packetSize, n) + err := gopacket.SerializeLayers(buffer, options, + ip.(gopacket.SerializableLayer), + &udp, + gopacket.Payload(payload[i:j]), + ) + if err != nil { + return nil, err + } + packet := buffer.Bytes() + packetCopy := make([]byte, len(packet)) + copy(packetCopy, packet) + packets = append(packets, packetCopy) + buffer.Clear() + i = j + } + return packets, nil + } } - type TcpBufferSettings struct { - ConnectTimeout time.Duration - ReadTimeout time.Duration - WriteTimeout time.Duration - // ReadPollTimeout time.Duration - // WritePollTimeout time.Duration - IdleTimeout time.Duration - ReadBufferByteCount int - SequenceBufferSize int - Mtu int - // the window size is the max amount of packet data in memory for each sequence - // TODO currently we do not enable window scale - // TODO this value is max 2^16 - WindowSize int - // the number of open sockets per user - // uses an lru cleanup where new sockets over the limit close old sockets - UserLimit int + ConnectTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration + // ReadPollTimeout time.Duration + // WritePollTimeout time.Duration + IdleTimeout time.Duration + ReadBufferByteCount int + SequenceBufferSize int + Mtu int + // the window size is the max amount of packet data in memory for each sequence + // TODO currently we do not enable window scale + // TODO this value is max 2^16 + WindowSize int + // the number of open sockets per user + // uses an lru cleanup where new sockets over the limit close old sockets + UserLimit int } - type Tcp4Buffer struct { - TcpBuffer[BufferId4] + TcpBuffer[BufferId4] } func NewTcp4Buffer(ctx context.Context, receiveCallback ReceivePacketFunction, - tcpBufferSettings *TcpBufferSettings) *Tcp4Buffer { - return &Tcp4Buffer{ - TcpBuffer: *newTcpBuffer[BufferId4](ctx, receiveCallback, tcpBufferSettings), - } -} - -func (self *Tcp4Buffer) send(source TransferPath, provideMode protocol.ProvideMode, - ipv4 *layers.IPv4, tcp *layers.TCP, timeout time.Duration) (bool, error) { - bufferId := NewBufferId4( - source, - ipv4.SrcIP, int(tcp.SrcPort), - ipv4.DstIP, int(tcp.DstPort), - ) - - return self.tcpSend( - bufferId, - ipv4.SrcIP, - ipv4.DstIP, - source, - provideMode, - 4, - tcp, - timeout, - ) + tcpBufferSettings *TcpBufferSettings) *Tcp4Buffer { + return &Tcp4Buffer{ + TcpBuffer: *newTcpBuffer[BufferId4](ctx, receiveCallback, tcpBufferSettings), + } +} + +func (self *Tcp4Buffer) send(source TransferPath, provideMode protocol.ProvideMode, + ipv4 *layers.IPv4, tcp *layers.TCP, timeout time.Duration) (bool, error) { + bufferId := NewBufferId4( + source, + ipv4.SrcIP, int(tcp.SrcPort), + ipv4.DstIP, int(tcp.DstPort), + ) + + return self.tcpSend( + bufferId, + ipv4.SrcIP, + ipv4.DstIP, + source, + provideMode, + 4, + tcp, + timeout, + ) } func (self *TcpSequence) Cancel() { - self.cancel() + self.cancel() } func (self *TcpSequence) Close() { - self.cancel() + self.cancel() } - type Tcp6Buffer struct { - TcpBuffer[BufferId6] + TcpBuffer[BufferId6] } func NewTcp6Buffer(ctx context.Context, receiveCallback ReceivePacketFunction, - tcpBufferSettings *TcpBufferSettings) *Tcp6Buffer { - return &Tcp6Buffer{ - TcpBuffer: *newTcpBuffer[BufferId6](ctx, receiveCallback, tcpBufferSettings), - } -} - -func (self *Tcp6Buffer) send(source TransferPath, provideMode protocol.ProvideMode, - ipv6 *layers.IPv6, tcp *layers.TCP, timeout time.Duration) (bool, error) { - bufferId := NewBufferId6( - source, - ipv6.SrcIP, int(tcp.SrcPort), - ipv6.DstIP, int(tcp.DstPort), - ) - - return self.tcpSend( - bufferId, - ipv6.SrcIP, - ipv6.DstIP, - source, - provideMode, - 6, - tcp, - timeout, - ) + tcpBufferSettings *TcpBufferSettings) *Tcp6Buffer { + return &Tcp6Buffer{ + TcpBuffer: *newTcpBuffer[BufferId6](ctx, receiveCallback, tcpBufferSettings), + } +} + +func (self *Tcp6Buffer) send(source TransferPath, provideMode protocol.ProvideMode, + ipv6 *layers.IPv6, tcp *layers.TCP, timeout time.Duration) (bool, error) { + bufferId := NewBufferId6( + source, + ipv6.SrcIP, int(tcp.SrcPort), + ipv6.DstIP, int(tcp.DstPort), + ) + + return self.tcpSend( + bufferId, + ipv6.SrcIP, + ipv6.DstIP, + source, + provideMode, + 6, + tcp, + timeout, + ) } - type TcpBuffer[BufferId comparable] struct { - ctx context.Context - receiveCallback ReceivePacketFunction - tcpBufferSettings *TcpBufferSettings + ctx context.Context + receiveCallback ReceivePacketFunction + tcpBufferSettings *TcpBufferSettings - mutex sync.Mutex + mutex sync.Mutex - sequences map[BufferId]*TcpSequence - sourceSequences map[TransferPath]map[BufferId]*TcpSequence + sequences map[BufferId]*TcpSequence + sourceSequences map[TransferPath]map[BufferId]*TcpSequence } func newTcpBuffer[BufferId comparable]( - ctx context.Context, - receiveCallback ReceivePacketFunction, - tcpBufferSettings *TcpBufferSettings, + ctx context.Context, + receiveCallback ReceivePacketFunction, + tcpBufferSettings *TcpBufferSettings, ) *TcpBuffer[BufferId] { - return &TcpBuffer[BufferId]{ - ctx: ctx, - receiveCallback: receiveCallback, - tcpBufferSettings: tcpBufferSettings, - sequences: map[BufferId]*TcpSequence{}, - sourceSequences: map[TransferPath]map[BufferId]*TcpSequence{}, - } + return &TcpBuffer[BufferId]{ + ctx: ctx, + receiveCallback: receiveCallback, + tcpBufferSettings: tcpBufferSettings, + sequences: map[BufferId]*TcpSequence{}, + sourceSequences: map[TransferPath]map[BufferId]*TcpSequence{}, + } } - func (self *TcpBuffer[BufferId]) tcpSend( - bufferId BufferId, - sourceIp net.IP, - destinationIp net.IP, - source TransferPath, - provideMode protocol.ProvideMode, - ipVersion int, - tcp *layers.TCP, - timeout time.Duration, + bufferId BufferId, + sourceIp net.IP, + destinationIp net.IP, + source TransferPath, + provideMode protocol.ProvideMode, + ipVersion int, + tcp *layers.TCP, + timeout time.Duration, ) (bool, error) { - initSequence := func()(*TcpSequence) { - self.mutex.Lock() - defer self.mutex.Unlock() - - if !tcp.SYN { - if sequence, ok := self.sequences[bufferId]; ok { - return sequence - } - // drop the packet; only create a new sequence on SYN - glog.V(2).Infof("[lnr]tcp drop no syn (%s)\n", tcpFlagsString(tcp)) - return nil - } - - // else new sequence - if sequence, ok := self.sequences[bufferId]; ok { - sequence.Cancel() - delete(self.sequences, bufferId) - sourceSequences := self.sourceSequences[sequence.source] - delete(sourceSequences, bufferId) - if 0 == len(sourceSequences) { - delete(self.sourceSequences, sequence.source) - } - } - - if 0 < self.tcpBufferSettings.UserLimit { - // limit the total connections per source to avoid blowing up the ulimit - if sourceSequences := self.sourceSequences[source]; self.tcpBufferSettings.UserLimit < len(sourceSequences) { - applyLruUserLimit(maps.Values(sourceSequences), self.tcpBufferSettings.UserLimit, func(sequence *TcpSequence)(bool) { - glog.Infof( - "[lnr]tcp limit source %s->%s\n", - source, - net.JoinHostPort( - sequence.destinationIp.String(), - strconv.Itoa(int(sequence.destinationPort)), - ), - ) - return true - }) - } - } - - sequence := NewTcpSequence( - self.ctx, - self.receiveCallback, - source, - ipVersion, - sourceIp, - tcp.SrcPort, - destinationIp, - tcp.DstPort, - self.tcpBufferSettings, - ) - self.sequences[bufferId] = sequence - go func() { - sequence.Run() - - self.mutex.Lock() - defer self.mutex.Unlock() - sequence.Close() - // clean up - if sequence == self.sequences[bufferId] { - delete(self.sequences, bufferId) - sourceSequences := self.sourceSequences[sequence.source] - delete(sourceSequences, bufferId) - if 0 == len(sourceSequences) { - delete(self.sourceSequences, sequence.source) - } - } - }() - return sequence - } - sendItem := &TcpSendItem{ - provideMode: provideMode, - tcp: tcp, - } - if sequence := initSequence(); sequence == nil { - // sequence does not exist and not a syn packet, drop - return false, nil - } else { - return sequence.send(sendItem, timeout) - } + initSequence := func() *TcpSequence { + self.mutex.Lock() + defer self.mutex.Unlock() + + if !tcp.SYN { + if sequence, ok := self.sequences[bufferId]; ok { + return sequence + } + // drop the packet; only create a new sequence on SYN + glog.V(2).Infof("[lnr]tcp drop no syn (%s)\n", tcpFlagsString(tcp)) + return nil + } + + // else new sequence + if sequence, ok := self.sequences[bufferId]; ok { + sequence.Cancel() + delete(self.sequences, bufferId) + sourceSequences := self.sourceSequences[sequence.source] + delete(sourceSequences, bufferId) + if 0 == len(sourceSequences) { + delete(self.sourceSequences, sequence.source) + } + } + + if 0 < self.tcpBufferSettings.UserLimit { + // limit the total connections per source to avoid blowing up the ulimit + if sourceSequences := self.sourceSequences[source]; self.tcpBufferSettings.UserLimit < len(sourceSequences) { + applyLruUserLimit(maps.Values(sourceSequences), self.tcpBufferSettings.UserLimit, func(sequence *TcpSequence) bool { + glog.Infof( + "[lnr]tcp limit source %s->%s\n", + source, + net.JoinHostPort( + sequence.destinationIp.String(), + strconv.Itoa(int(sequence.destinationPort)), + ), + ) + return true + }) + } + } + + sequence := NewTcpSequence( + self.ctx, + self.receiveCallback, + source, + ipVersion, + sourceIp, + tcp.SrcPort, + destinationIp, + tcp.DstPort, + self.tcpBufferSettings, + ) + self.sequences[bufferId] = sequence + go func() { + sequence.Run() + + self.mutex.Lock() + defer self.mutex.Unlock() + sequence.Close() + // clean up + if sequence == self.sequences[bufferId] { + delete(self.sequences, bufferId) + sourceSequences := self.sourceSequences[sequence.source] + delete(sourceSequences, bufferId) + if 0 == len(sourceSequences) { + delete(self.sourceSequences, sequence.source) + } + } + }() + return sequence + } + sendItem := &TcpSendItem{ + provideMode: provideMode, + tcp: tcp, + } + if sequence := initSequence(); sequence == nil { + // sequence does not exist and not a syn packet, drop + return false, nil + } else { + return sequence.send(sendItem, timeout) + } } /* @@ -1146,1468 +1117,1444 @@ is assumed to never require retransmits. The retrasmit logic is not implemented. This is a safe assumption when moving packets from local raw socket to the UNAT via `transfer`, which is lossless and in-order. - */ type TcpSequence struct { - ctx context.Context - cancel context.CancelFunc - - receiveCallback ReceivePacketFunction + ctx context.Context + cancel context.CancelFunc - tcpBufferSettings *TcpBufferSettings + receiveCallback ReceivePacketFunction - sendItems chan *TcpSendItem + tcpBufferSettings *TcpBufferSettings - idleCondition *IdleCondition + sendItems chan *TcpSendItem - ConnectionState + idleCondition *IdleCondition + + ConnectionState } func NewTcpSequence(ctx context.Context, receiveCallback ReceivePacketFunction, - source TransferPath, - ipVersion int, - sourceIp net.IP, sourcePort layers.TCPPort, - destinationIp net.IP, destinationPort layers.TCPPort, - tcpBufferSettings *TcpBufferSettings) *TcpSequence { - cancelCtx, cancel := context.WithCancel(ctx) - - var windowSize uint16 - if math.MaxUint16 < tcpBufferSettings.WindowSize { - windowSize = math.MaxUint16 - } else { - windowSize = uint16(tcpBufferSettings.WindowSize) - } - - connectionState := ConnectionState{ - source: source, - ipVersion: ipVersion, - sourceIp: sourceIp, - sourcePort: sourcePort, - destinationIp: destinationIp, - destinationPort: destinationPort, - // the window size starts at the fixed value - windowSize: windowSize, - userLimited: *newUserLimited(), - } - return &TcpSequence{ - ctx: cancelCtx, - cancel: cancel, - receiveCallback: receiveCallback, - tcpBufferSettings: tcpBufferSettings, - sendItems: make(chan *TcpSendItem, tcpBufferSettings.SequenceBufferSize), - idleCondition: NewIdleCondition(), - ConnectionState: connectionState, - } + source TransferPath, + ipVersion int, + sourceIp net.IP, sourcePort layers.TCPPort, + destinationIp net.IP, destinationPort layers.TCPPort, + tcpBufferSettings *TcpBufferSettings) *TcpSequence { + cancelCtx, cancel := context.WithCancel(ctx) + + var windowSize uint16 + if math.MaxUint16 < tcpBufferSettings.WindowSize { + windowSize = math.MaxUint16 + } else { + windowSize = uint16(tcpBufferSettings.WindowSize) + } + + connectionState := ConnectionState{ + source: source, + ipVersion: ipVersion, + sourceIp: sourceIp, + sourcePort: sourcePort, + destinationIp: destinationIp, + destinationPort: destinationPort, + // the window size starts at the fixed value + windowSize: windowSize, + userLimited: *newUserLimited(), + } + return &TcpSequence{ + ctx: cancelCtx, + cancel: cancel, + receiveCallback: receiveCallback, + tcpBufferSettings: tcpBufferSettings, + sendItems: make(chan *TcpSendItem, tcpBufferSettings.SequenceBufferSize), + idleCondition: NewIdleCondition(), + ConnectionState: connectionState, + } } func (self *TcpSequence) send(sendItem *TcpSendItem, timeout time.Duration) (bool, error) { - if !self.idleCondition.UpdateOpen() { - return false, nil - } - defer self.idleCondition.UpdateClose() - - select { - case <- self.ctx.Done(): - return false, errors.New("Done.") - default: - } - - if timeout < 0 { - select { - case <- self.ctx.Done(): - return false, errors.New("Done.") - case self.sendItems <- sendItem: - return true, nil - } - } else if timeout == 0 { - select { - case <- self.ctx.Done(): - return false, errors.New("Done.") - case self.sendItems <- sendItem: - return true, nil - default: - return false, nil - } - } else { - select { - case <- self.ctx.Done(): - return false, errors.New("Done.") - case self.sendItems <- sendItem: - return true, nil - case <- time.After(timeout): - return false, nil - } - } + if !self.idleCondition.UpdateOpen() { + return false, nil + } + defer self.idleCondition.UpdateClose() + + select { + case <-self.ctx.Done(): + return false, errors.New("Done.") + default: + } + + if timeout < 0 { + select { + case <-self.ctx.Done(): + return false, errors.New("Done.") + case self.sendItems <- sendItem: + return true, nil + } + } else if timeout == 0 { + select { + case <-self.ctx.Done(): + return false, errors.New("Done.") + case self.sendItems <- sendItem: + return true, nil + default: + return false, nil + } + } else { + select { + case <-self.ctx.Done(): + return false, errors.New("Done.") + case self.sendItems <- sendItem: + return true, nil + case <-time.After(timeout): + return false, nil + } + } } func (self *TcpSequence) Run() { - defer self.cancel() - - receive := func(packet []byte) { - self.receiveCallback(self.source, IpProtocolTcp, packet) - } - - closed := false - // send a final FIN+ACK - defer func() { - if closed { - glog.V(2).Infof("[r]closed gracefully\n") - } else { - glog.V(2).Infof("[r]closed unexpected sending RST\n") - var packet []byte - var err error - func() { - self.mutex.Lock() - defer self.mutex.Unlock() - - packet, err = self.RstAck() - }() - if err == nil { - receive(packet) - } - } - }() - - - for syn := false; !syn; { - select { - case <- self.ctx.Done(): - return - case sendItem := <- self.sendItems: - glog.V(2).Infof("[init]send(%d)\n", len(sendItem.tcp.BaseLayer.Payload)) - // the first packet must be a syn - if sendItem.tcp.SYN { - glog.V(2).Infof("[init]SYN\n") - - var packet []byte - var err error - func() { - self.mutex.Lock() - defer self.mutex.Unlock() - - // sendSeq is the next expected sequence number - // SYN and FIN consume one - self.sendSeq = sendItem.tcp.Seq + 1 - // start the send seq at send seq - // this is arbitrary, and since there is no transport security risk back to sender is fine - self.receiveSeq = sendItem.tcp.Seq - self.receiveSeqAck = sendItem.tcp.Seq - self.receiveWindowSize = sendItem.tcp.Window - packet, err = self.SynAck() - self.receiveSeq += 1 - }() - if err == nil { - glog.V(2).Infof("[init]receive SYN+ACK\n") - receive(packet) - } - - syn = true - } else { - // an ACK here could be for a previous FIN - glog.V(2).Infof("[init]waiting for SYN (%s)\n", tcpFlagsString(sendItem.tcp)) - } - } - } - - glog.V(2).Infof("[init]tcp connect\n") - socket, err := net.DialTimeout( - "tcp", - self.DestinationAuthority(), - self.tcpBufferSettings.ConnectTimeout, - ) - if err != nil { - glog.Infof("[init]tcp connect error = %s\n", err) - return - } - defer socket.Close() - - // tcpSocket := socket.(*net.TCPConn) - // tcpSocket.SetNoDelay(false) - - self.UpdateLastActivityTime() - glog.V(2).Infof("[init]connect success\n") - - receiveAckCond := sync.NewCond(&self.mutex) - defer func() { - self.mutex.Lock() - defer self.mutex.Unlock() - - receiveAckCond.Broadcast() - }() - - go func() { - defer self.cancel() - - buffer := make([]byte, self.tcpBufferSettings.ReadBufferByteCount) - - - for forwardIter := uint64(0); ; forwardIter += 1 { - select { - case <- self.ctx.Done(): - return - default: - } - - readTimeout := time.Now().Add(self.tcpBufferSettings.ReadTimeout) - socket.SetReadDeadline(readTimeout) - - n, err := socket.Read(buffer) - - if err != nil { - glog.Infof("[f%d]tcp receive error = %s\n", forwardIter, err) - } - - if 0 < n { - self.UpdateLastActivityTime() - - // since the transfer from local to remove is lossless and preserves order, - // do not worry about retransmits - var packets [][]byte - var packetsErr error - func() { - self.mutex.Lock() - defer self.mutex.Unlock() - - packets, packetsErr = self.DataPackets(buffer, n, self.tcpBufferSettings.Mtu) - if packetsErr != nil { - glog.Infof("[f%d]tcp receive packets error = %s\n", forwardIter, packetsErr) - return - } - - if 1 < len(packets) { - glog.V(2).Infof("[f%d]tcp receive segmented packets %d\n", forwardIter, len(packets)) - } - glog.V(2).Infof("[f%d]tcp receive %d %d %d\n", forwardIter, n, len(packets), self.receiveSeq) - - for uint32(self.receiveWindowSize) < self.receiveSeq - self.receiveSeqAck + uint32(n) { - glog.V(2).Infof("[f%d]tcp receive window wait\n", forwardIter) - receiveAckCond.Wait() - select { - case <- self.ctx.Done(): - return - default: - } - } - - self.receiveSeq += uint32(n) - }() - if packetsErr != nil { - return - } - - select { - case <- self.ctx.Done(): - return - default: - } - - for _, packet := range packets { - receive(packet) - } - } - - if err != nil { - if err == io.EOF { - // closed (FIN) - // propagate the FIN and close the sequence - glog.V(2).Infof("[final]FIN\n") - var packet []byte - var err error - func() { - self.mutex.Lock() - defer self.mutex.Unlock() - - packet, err = self.FinAck() - self.receiveSeq += 1 - }() - if err == nil { - closed = true - receive(packet) - } - return - } else if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - glog.V(2).Infof("[f%d]timeout\n", forwardIter) - return - } else { - // some other error - return - } - } - } - }() - - go func() { - defer self.cancel() - - for sendIter := 0; ; sendIter += 1 { - checkpointId := self.idleCondition.Checkpoint() - select { - case <- self.ctx.Done(): - return - case sendItem := <- self.sendItems: - if glog.V(2) { - if "ACK" != tcpFlagsString(sendItem.tcp) { - glog.Infof("[r%d]receive(%d %s)\n", sendIter, len(sendItem.tcp.Payload), tcpFlagsString(sendItem.tcp)) - } - } - - drop := false - seq := 0 - - func() { - self.mutex.Lock() - defer self.mutex.Unlock() - - if self.sendSeq != sendItem.tcp.Seq { - // a retransmit - // since the transfer from local to remote is lossless and preserves order, - // the packet is already pending. Ignore. - drop = true - } else if sendItem.tcp.ACK { - // acks are reliably delivered (see above) - // we do not need to resend receive packets on missing acks - // note the window size can be be adjusted at any time for the same receive seq number, - // e.g. ->0 then ->full on receiver full - if self.receiveSeqAck <= sendItem.tcp.Ack { - self.receiveWindowSize = sendItem.tcp.Window - self.receiveSeqAck = sendItem.tcp.Ack - receiveAckCond.Broadcast() - } - } - }() - - if drop { - continue - } - - if sendItem.tcp.FIN { - glog.V(2).Infof("[r%d]FIN\n", sendIter) - seq += 1 - } - - writeEndTime := time.Now().Add(self.tcpBufferSettings.WriteTimeout) - - payload := sendItem.tcp.Payload - seq += len(payload) - for i := 0; i < len(payload); { - select { - case <- self.ctx.Done(): - return - default: - } - - socket.SetWriteDeadline(writeEndTime) - n, err := socket.Write(payload[i:]) - - if err == nil { - glog.V(2).Infof("[f%d]tcp forward %d\n", sendIter, n) - } else { - glog.Infof("[f%d]tcp forward %d error = %s\n", sendIter, n, err) - } - - if 0 < n { - self.UpdateLastActivityTime() - - j := i - i += n - glog.V(2).Infof("[f%d]tcp forward %d/%d -> %d/%d +%d\n", sendIter, j, len(payload), i, len(payload), n) - } - - if err != nil { - if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - return - } else { - // some other error - return - } - } - } - - - if 0 < seq { - var packet []byte - var err error - func() { - self.mutex.Lock() - defer self.mutex.Unlock() - - self.sendSeq += uint32(seq) - packet, err = self.PureAck() - }() - if err == nil { - receive(packet) - } - } - - - - - if sendItem.tcp.FIN { - // close the socket to propage the FIN and close the sequence - socket.Close() - } - - if sendItem.tcp.RST { - // a RST typically appears for a bad TCP segment - glog.V(2).Infof("[r%d]RST\n", sendIter) - return - } - - case <- time.After(self.tcpBufferSettings.IdleTimeout): - if self.idleCondition.Close(checkpointId) { - // close the sequence - glog.V(2).Infof("[r%d]timeout\n", sendIter) - return - } - // else there pending updates - } - } - }() - - select { - case <- self.ctx.Done(): - } + defer self.cancel() + + receive := func(packet []byte) { + self.receiveCallback(self.source, IpProtocolTcp, packet) + } + + closed := false + // send a final FIN+ACK + defer func() { + if closed { + glog.V(2).Infof("[r]closed gracefully\n") + } else { + glog.V(2).Infof("[r]closed unexpected sending RST\n") + var packet []byte + var err error + func() { + self.mutex.Lock() + defer self.mutex.Unlock() + + packet, err = self.RstAck() + }() + if err == nil { + receive(packet) + } + } + }() + + for syn := false; !syn; { + select { + case <-self.ctx.Done(): + return + case sendItem := <-self.sendItems: + glog.V(2).Infof("[init]send(%d)\n", len(sendItem.tcp.BaseLayer.Payload)) + // the first packet must be a syn + if sendItem.tcp.SYN { + glog.V(2).Infof("[init]SYN\n") + + var packet []byte + var err error + func() { + self.mutex.Lock() + defer self.mutex.Unlock() + + // sendSeq is the next expected sequence number + // SYN and FIN consume one + self.sendSeq = sendItem.tcp.Seq + 1 + // start the send seq at send seq + // this is arbitrary, and since there is no transport security risk back to sender is fine + self.receiveSeq = sendItem.tcp.Seq + self.receiveSeqAck = sendItem.tcp.Seq + self.receiveWindowSize = sendItem.tcp.Window + packet, err = self.SynAck() + self.receiveSeq += 1 + }() + if err == nil { + glog.V(2).Infof("[init]receive SYN+ACK\n") + receive(packet) + } + + syn = true + } else { + // an ACK here could be for a previous FIN + glog.V(2).Infof("[init]waiting for SYN (%s)\n", tcpFlagsString(sendItem.tcp)) + } + } + } + + glog.V(2).Infof("[init]tcp connect\n") + socket, err := net.DialTimeout( + "tcp", + self.DestinationAuthority(), + self.tcpBufferSettings.ConnectTimeout, + ) + if err != nil { + glog.Infof("[init]tcp connect error = %s\n", err) + return + } + defer socket.Close() + + // tcpSocket := socket.(*net.TCPConn) + // tcpSocket.SetNoDelay(false) + + self.UpdateLastActivityTime() + glog.V(2).Infof("[init]connect success\n") + + receiveAckCond := sync.NewCond(&self.mutex) + defer func() { + self.mutex.Lock() + defer self.mutex.Unlock() + + receiveAckCond.Broadcast() + }() + + go func() { + defer self.cancel() + + buffer := make([]byte, self.tcpBufferSettings.ReadBufferByteCount) + + for forwardIter := uint64(0); ; forwardIter += 1 { + select { + case <-self.ctx.Done(): + return + default: + } + + readTimeout := time.Now().Add(self.tcpBufferSettings.ReadTimeout) + socket.SetReadDeadline(readTimeout) + + n, err := socket.Read(buffer) + + if err != nil { + glog.Infof("[f%d]tcp receive error = %s\n", forwardIter, err) + } + + if 0 < n { + self.UpdateLastActivityTime() + + // since the transfer from local to remove is lossless and preserves order, + // do not worry about retransmits + var packets [][]byte + var packetsErr error + func() { + self.mutex.Lock() + defer self.mutex.Unlock() + + packets, packetsErr = self.DataPackets(buffer, n, self.tcpBufferSettings.Mtu) + if packetsErr != nil { + glog.Infof("[f%d]tcp receive packets error = %s\n", forwardIter, packetsErr) + return + } + + if 1 < len(packets) { + glog.V(2).Infof("[f%d]tcp receive segmented packets %d\n", forwardIter, len(packets)) + } + glog.V(2).Infof("[f%d]tcp receive %d %d %d\n", forwardIter, n, len(packets), self.receiveSeq) + + for uint32(self.receiveWindowSize) < self.receiveSeq-self.receiveSeqAck+uint32(n) { + glog.V(2).Infof("[f%d]tcp receive window wait\n", forwardIter) + receiveAckCond.Wait() + select { + case <-self.ctx.Done(): + return + default: + } + } + + self.receiveSeq += uint32(n) + }() + if packetsErr != nil { + return + } + + select { + case <-self.ctx.Done(): + return + default: + } + + for _, packet := range packets { + receive(packet) + } + } + + if err != nil { + if err == io.EOF { + // closed (FIN) + // propagate the FIN and close the sequence + glog.V(2).Infof("[final]FIN\n") + var packet []byte + var err error + func() { + self.mutex.Lock() + defer self.mutex.Unlock() + + packet, err = self.FinAck() + self.receiveSeq += 1 + }() + if err == nil { + closed = true + receive(packet) + } + return + } else if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + glog.V(2).Infof("[f%d]timeout\n", forwardIter) + return + } else { + // some other error + return + } + } + } + }() + + go func() { + defer self.cancel() + + for sendIter := 0; ; sendIter += 1 { + checkpointId := self.idleCondition.Checkpoint() + select { + case <-self.ctx.Done(): + return + case sendItem := <-self.sendItems: + if glog.V(2) { + if "ACK" != tcpFlagsString(sendItem.tcp) { + glog.Infof("[r%d]receive(%d %s)\n", sendIter, len(sendItem.tcp.Payload), tcpFlagsString(sendItem.tcp)) + } + } + + drop := false + seq := 0 + + func() { + self.mutex.Lock() + defer self.mutex.Unlock() + + if self.sendSeq != sendItem.tcp.Seq { + // a retransmit + // since the transfer from local to remote is lossless and preserves order, + // the packet is already pending. Ignore. + drop = true + } else if sendItem.tcp.ACK { + // acks are reliably delivered (see above) + // we do not need to resend receive packets on missing acks + // note the window size can be be adjusted at any time for the same receive seq number, + // e.g. ->0 then ->full on receiver full + if self.receiveSeqAck <= sendItem.tcp.Ack { + self.receiveWindowSize = sendItem.tcp.Window + self.receiveSeqAck = sendItem.tcp.Ack + receiveAckCond.Broadcast() + } + } + }() + + if drop { + continue + } + + if sendItem.tcp.FIN { + glog.V(2).Infof("[r%d]FIN\n", sendIter) + seq += 1 + } + + writeEndTime := time.Now().Add(self.tcpBufferSettings.WriteTimeout) + + payload := sendItem.tcp.Payload + seq += len(payload) + for i := 0; i < len(payload); { + select { + case <-self.ctx.Done(): + return + default: + } + + socket.SetWriteDeadline(writeEndTime) + n, err := socket.Write(payload[i:]) + + if err == nil { + glog.V(2).Infof("[f%d]tcp forward %d\n", sendIter, n) + } else { + glog.Infof("[f%d]tcp forward %d error = %s\n", sendIter, n, err) + } + + if 0 < n { + self.UpdateLastActivityTime() + + j := i + i += n + glog.V(2).Infof("[f%d]tcp forward %d/%d -> %d/%d +%d\n", sendIter, j, len(payload), i, len(payload), n) + } + + if err != nil { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + return + } else { + // some other error + return + } + } + } + + if 0 < seq { + var packet []byte + var err error + func() { + self.mutex.Lock() + defer self.mutex.Unlock() + + self.sendSeq += uint32(seq) + packet, err = self.PureAck() + }() + if err == nil { + receive(packet) + } + } + + if sendItem.tcp.FIN { + // close the socket to propage the FIN and close the sequence + socket.Close() + } + + if sendItem.tcp.RST { + // a RST typically appears for a bad TCP segment + glog.V(2).Infof("[r%d]RST\n", sendIter) + return + } + + case <-time.After(self.tcpBufferSettings.IdleTimeout): + if self.idleCondition.Close(checkpointId) { + // close the sequence + glog.V(2).Infof("[r%d]timeout\n", sendIter) + return + } + // else there pending updates + } + } + }() + + select { + case <-self.ctx.Done(): + } } type TcpSendItem struct { - provideMode protocol.ProvideMode - tcp *layers.TCP + provideMode protocol.ProvideMode + tcp *layers.TCP } type ConnectionState struct { - source TransferPath - ipVersion int - sourceIp net.IP - sourcePort layers.TCPPort - destinationIp net.IP - destinationPort layers.TCPPort + source TransferPath + ipVersion int + sourceIp net.IP + sourcePort layers.TCPPort + destinationIp net.IP + destinationPort layers.TCPPort - mutex sync.Mutex + mutex sync.Mutex - sendSeq uint32 - receiveSeq uint32 - receiveSeqAck uint32 - receiveWindowSize uint16 - windowSize uint16 + sendSeq uint32 + receiveSeq uint32 + receiveSeqAck uint32 + receiveWindowSize uint16 + windowSize uint16 - userLimited + userLimited } func (self *ConnectionState) SourceAuthority() string { - return net.JoinHostPort( - self.sourceIp.String(), - strconv.Itoa(int(self.sourcePort)), - ) + return net.JoinHostPort( + self.sourceIp.String(), + strconv.Itoa(int(self.sourcePort)), + ) } func (self *ConnectionState) DestinationAuthority() string { - return net.JoinHostPort( - self.destinationIp.String(), - strconv.Itoa(int(self.destinationPort)), - ) + return net.JoinHostPort( + self.destinationIp.String(), + strconv.Itoa(int(self.destinationPort)), + ) } func (self *ConnectionState) SynAck() ([]byte, error) { - headerSize := 0 - var ip gopacket.NetworkLayer - switch self.ipVersion { - case 4: - ip = &layers.IPv4{ - Version: 4, - TTL: 64, - SrcIP: self.destinationIp, - DstIP: self.sourceIp, - Protocol: layers.IPProtocolTCP, - } - headerSize += Ipv4HeaderSizeWithoutExtensions - case 6: - ip = &layers.IPv6{ - Version: 6, - HopLimit: 64, - SrcIP: self.destinationIp, - DstIP: self.sourceIp, - NextHeader: layers.IPProtocolTCP, - } - headerSize += Ipv6HeaderSize - } - - tcp := layers.TCP{ - SrcPort: self.destinationPort, - DstPort: self.sourcePort, - Seq: self.receiveSeq, - Ack: self.sendSeq, - ACK: true, - SYN: true, - Window: self.windowSize, - // TODO window scale - // https://datatracker.ietf.org/doc/html/rfc1323#page-8 - } - tcp.SetNetworkLayerForChecksum(ip) - headerSize += TcpHeaderSizeWithoutExtensions - - options := gopacket.SerializeOptions{ - ComputeChecksums: true, - FixLengths: true, - } - - buffer := gopacket.NewSerializeBufferExpectedSize(headerSize, 0) - - err := gopacket.SerializeLayers(buffer, options, - ip.(gopacket.SerializableLayer), - &tcp, - ) - - if err != nil { - return nil, err - } - packet := buffer.Bytes() - return packet, nil + headerSize := 0 + var ip gopacket.NetworkLayer + switch self.ipVersion { + case 4: + ip = &layers.IPv4{ + Version: 4, + TTL: 64, + SrcIP: self.destinationIp, + DstIP: self.sourceIp, + Protocol: layers.IPProtocolTCP, + } + headerSize += Ipv4HeaderSizeWithoutExtensions + case 6: + ip = &layers.IPv6{ + Version: 6, + HopLimit: 64, + SrcIP: self.destinationIp, + DstIP: self.sourceIp, + NextHeader: layers.IPProtocolTCP, + } + headerSize += Ipv6HeaderSize + } + + tcp := layers.TCP{ + SrcPort: self.destinationPort, + DstPort: self.sourcePort, + Seq: self.receiveSeq, + Ack: self.sendSeq, + ACK: true, + SYN: true, + Window: self.windowSize, + // TODO window scale + // https://datatracker.ietf.org/doc/html/rfc1323#page-8 + } + tcp.SetNetworkLayerForChecksum(ip) + headerSize += TcpHeaderSizeWithoutExtensions + + options := gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + } + + buffer := gopacket.NewSerializeBufferExpectedSize(headerSize, 0) + + err := gopacket.SerializeLayers(buffer, options, + ip.(gopacket.SerializableLayer), + &tcp, + ) + + if err != nil { + return nil, err + } + packet := buffer.Bytes() + return packet, nil } func (self *ConnectionState) PureAck() ([]byte, error) { - headerSize := 0 - var ip gopacket.NetworkLayer - switch self.ipVersion { - case 4: - ip = &layers.IPv4{ - Version: 4, - TTL: 64, - SrcIP: self.destinationIp, - DstIP: self.sourceIp, - Protocol: layers.IPProtocolTCP, - } - headerSize += Ipv4HeaderSizeWithoutExtensions - case 6: - ip = &layers.IPv6{ - Version: 6, - HopLimit: 64, - SrcIP: self.destinationIp, - DstIP: self.sourceIp, - NextHeader: layers.IPProtocolTCP, - } - headerSize += Ipv6HeaderSize - } - - tcp := layers.TCP{ - SrcPort: self.destinationPort, - DstPort: self.sourcePort, - Seq: self.receiveSeq, - Ack: self.sendSeq, - ACK: true, - Window: self.windowSize, - } - tcp.SetNetworkLayerForChecksum(ip) - headerSize += TcpHeaderSizeWithoutExtensions - - options := gopacket.SerializeOptions{ - ComputeChecksums: true, - FixLengths: true, - } - - buffer := gopacket.NewSerializeBufferExpectedSize(headerSize, 0) - - err := gopacket.SerializeLayers(buffer, options, - ip.(gopacket.SerializableLayer), - &tcp, - ) - - if err != nil { - return nil, err - } - packet := buffer.Bytes() - return packet, nil + headerSize := 0 + var ip gopacket.NetworkLayer + switch self.ipVersion { + case 4: + ip = &layers.IPv4{ + Version: 4, + TTL: 64, + SrcIP: self.destinationIp, + DstIP: self.sourceIp, + Protocol: layers.IPProtocolTCP, + } + headerSize += Ipv4HeaderSizeWithoutExtensions + case 6: + ip = &layers.IPv6{ + Version: 6, + HopLimit: 64, + SrcIP: self.destinationIp, + DstIP: self.sourceIp, + NextHeader: layers.IPProtocolTCP, + } + headerSize += Ipv6HeaderSize + } + + tcp := layers.TCP{ + SrcPort: self.destinationPort, + DstPort: self.sourcePort, + Seq: self.receiveSeq, + Ack: self.sendSeq, + ACK: true, + Window: self.windowSize, + } + tcp.SetNetworkLayerForChecksum(ip) + headerSize += TcpHeaderSizeWithoutExtensions + + options := gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + } + + buffer := gopacket.NewSerializeBufferExpectedSize(headerSize, 0) + + err := gopacket.SerializeLayers(buffer, options, + ip.(gopacket.SerializableLayer), + &tcp, + ) + + if err != nil { + return nil, err + } + packet := buffer.Bytes() + return packet, nil } func (self *ConnectionState) FinAck() ([]byte, error) { - headerSize := 0 - var ip gopacket.NetworkLayer - switch self.ipVersion { - case 4: - ip = &layers.IPv4{ - Version: 4, - TTL: 64, - SrcIP: self.destinationIp, - DstIP: self.sourceIp, - Protocol: layers.IPProtocolTCP, - } - headerSize += Ipv4HeaderSizeWithoutExtensions - case 6: - ip = &layers.IPv6{ - Version: 6, - HopLimit: 64, - SrcIP: self.destinationIp, - DstIP: self.sourceIp, - NextHeader: layers.IPProtocolTCP, - } - headerSize += Ipv6HeaderSize - } - - tcp := layers.TCP{ - SrcPort: self.destinationPort, - DstPort: self.sourcePort, - Seq: self.receiveSeq, - Ack: self.sendSeq, - ACK: true, - FIN: true, - Window: self.windowSize, - } - tcp.SetNetworkLayerForChecksum(ip) - headerSize += TcpHeaderSizeWithoutExtensions - - options := gopacket.SerializeOptions{ - ComputeChecksums: true, - FixLengths: true, - } - - buffer := gopacket.NewSerializeBufferExpectedSize(headerSize, 0) - - err := gopacket.SerializeLayers(buffer, options, - ip.(gopacket.SerializableLayer), - &tcp, - ) - - if err != nil { - return nil, err - } - packet := buffer.Bytes() - return packet, nil + headerSize := 0 + var ip gopacket.NetworkLayer + switch self.ipVersion { + case 4: + ip = &layers.IPv4{ + Version: 4, + TTL: 64, + SrcIP: self.destinationIp, + DstIP: self.sourceIp, + Protocol: layers.IPProtocolTCP, + } + headerSize += Ipv4HeaderSizeWithoutExtensions + case 6: + ip = &layers.IPv6{ + Version: 6, + HopLimit: 64, + SrcIP: self.destinationIp, + DstIP: self.sourceIp, + NextHeader: layers.IPProtocolTCP, + } + headerSize += Ipv6HeaderSize + } + + tcp := layers.TCP{ + SrcPort: self.destinationPort, + DstPort: self.sourcePort, + Seq: self.receiveSeq, + Ack: self.sendSeq, + ACK: true, + FIN: true, + Window: self.windowSize, + } + tcp.SetNetworkLayerForChecksum(ip) + headerSize += TcpHeaderSizeWithoutExtensions + + options := gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + } + + buffer := gopacket.NewSerializeBufferExpectedSize(headerSize, 0) + + err := gopacket.SerializeLayers(buffer, options, + ip.(gopacket.SerializableLayer), + &tcp, + ) + + if err != nil { + return nil, err + } + packet := buffer.Bytes() + return packet, nil } func (self *ConnectionState) RstAck() ([]byte, error) { - headerSize := 0 - var ip gopacket.NetworkLayer - switch self.ipVersion { - case 4: - ip = &layers.IPv4{ - Version: 4, - TTL: 64, - SrcIP: self.destinationIp, - DstIP: self.sourceIp, - Protocol: layers.IPProtocolTCP, - } - headerSize += Ipv4HeaderSizeWithoutExtensions - case 6: - ip = &layers.IPv6{ - Version: 6, - HopLimit: 64, - SrcIP: self.destinationIp, - DstIP: self.sourceIp, - NextHeader: layers.IPProtocolTCP, - } - headerSize += Ipv6HeaderSize - } - - tcp := layers.TCP{ - SrcPort: self.destinationPort, - DstPort: self.sourcePort, - Seq: self.receiveSeq, - Ack: self.sendSeq, - ACK: true, - RST: true, - Window: self.windowSize, - } - tcp.SetNetworkLayerForChecksum(ip) - headerSize += TcpHeaderSizeWithoutExtensions - - options := gopacket.SerializeOptions{ - ComputeChecksums: true, - FixLengths: true, - } - - buffer := gopacket.NewSerializeBufferExpectedSize(headerSize, 0) - - err := gopacket.SerializeLayers(buffer, options, - ip.(gopacket.SerializableLayer), - &tcp, - ) - - if err != nil { - return nil, err - } - packet := buffer.Bytes() - return packet, nil + headerSize := 0 + var ip gopacket.NetworkLayer + switch self.ipVersion { + case 4: + ip = &layers.IPv4{ + Version: 4, + TTL: 64, + SrcIP: self.destinationIp, + DstIP: self.sourceIp, + Protocol: layers.IPProtocolTCP, + } + headerSize += Ipv4HeaderSizeWithoutExtensions + case 6: + ip = &layers.IPv6{ + Version: 6, + HopLimit: 64, + SrcIP: self.destinationIp, + DstIP: self.sourceIp, + NextHeader: layers.IPProtocolTCP, + } + headerSize += Ipv6HeaderSize + } + + tcp := layers.TCP{ + SrcPort: self.destinationPort, + DstPort: self.sourcePort, + Seq: self.receiveSeq, + Ack: self.sendSeq, + ACK: true, + RST: true, + Window: self.windowSize, + } + tcp.SetNetworkLayerForChecksum(ip) + headerSize += TcpHeaderSizeWithoutExtensions + + options := gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + } + + buffer := gopacket.NewSerializeBufferExpectedSize(headerSize, 0) + + err := gopacket.SerializeLayers(buffer, options, + ip.(gopacket.SerializableLayer), + &tcp, + ) + + if err != nil { + return nil, err + } + packet := buffer.Bytes() + return packet, nil } func (self *ConnectionState) DataPackets(payload []byte, n int, mtu int) ([][]byte, error) { - headerSize := 0 - var ip gopacket.NetworkLayer - switch self.ipVersion { - case 4: - ip = &layers.IPv4{ - Version: 4, - TTL: 64, - SrcIP: self.destinationIp, - DstIP: self.sourceIp, - Protocol: layers.IPProtocolTCP, - } - headerSize += Ipv4HeaderSizeWithoutExtensions - case 6: - ip = &layers.IPv6{ - Version: 6, - HopLimit: 64, - SrcIP: self.destinationIp, - DstIP: self.sourceIp, - NextHeader: layers.IPProtocolTCP, - } - headerSize += Ipv6HeaderSize - } - - tcp := layers.TCP{ - SrcPort: self.destinationPort, - DstPort: self.sourcePort, - Seq: self.receiveSeq, - Ack: self.sendSeq, - ACK: true, - Window: self.windowSize, - } - tcp.SetNetworkLayerForChecksum(ip) - headerSize += TcpHeaderSizeWithoutExtensions - - options := gopacket.SerializeOptions{ - ComputeChecksums: true, - FixLengths: true, - } - - if debugVerifyHeaders { - buffer := gopacket.NewSerializeBufferExpectedSize(headerSize, 0) - err := gopacket.SerializeLayers(buffer, options, - ip.(gopacket.SerializableLayer), - &tcp, - ) - if err != nil { - return nil, err - } - packetHeaders := buffer.Bytes() - if headerSize != len(packetHeaders) { - return nil, errors.New(fmt.Sprintf("Header check failed %d <> %d", headerSize, len(packetHeaders))) - } - } - - if headerSize + n <= mtu { - buffer := gopacket.NewSerializeBufferExpectedSize(headerSize + n, 0) - err := gopacket.SerializeLayers(buffer, options, - ip.(gopacket.SerializableLayer), - &tcp, - gopacket.Payload(payload[0:n]), - ) - if err != nil { - return nil, err - } - packet := buffer.Bytes() - return [][]byte{packet}, nil - } else { - // fragment - buffer := gopacket.NewSerializeBufferExpectedSize(mtu, 0) - packetSize := mtu - headerSize - packets := [][]byte{} - for i := 0; i < n; { - j := min(i + packetSize, n) - tcp.Seq = self.receiveSeq + uint32(i) - err := gopacket.SerializeLayers(buffer, options, - ip.(gopacket.SerializableLayer), - &tcp, - gopacket.Payload(payload[i:j]), - ) - if err != nil { - return nil, err - } - packet := buffer.Bytes() - packetCopy := make([]byte, len(packet)) - copy(packetCopy, packet) - packets = append(packets, packetCopy) - buffer.Clear() - i = j - } - return packets, nil - } + headerSize := 0 + var ip gopacket.NetworkLayer + switch self.ipVersion { + case 4: + ip = &layers.IPv4{ + Version: 4, + TTL: 64, + SrcIP: self.destinationIp, + DstIP: self.sourceIp, + Protocol: layers.IPProtocolTCP, + } + headerSize += Ipv4HeaderSizeWithoutExtensions + case 6: + ip = &layers.IPv6{ + Version: 6, + HopLimit: 64, + SrcIP: self.destinationIp, + DstIP: self.sourceIp, + NextHeader: layers.IPProtocolTCP, + } + headerSize += Ipv6HeaderSize + } + + tcp := layers.TCP{ + SrcPort: self.destinationPort, + DstPort: self.sourcePort, + Seq: self.receiveSeq, + Ack: self.sendSeq, + ACK: true, + Window: self.windowSize, + } + tcp.SetNetworkLayerForChecksum(ip) + headerSize += TcpHeaderSizeWithoutExtensions + + options := gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + } + + if debugVerifyHeaders { + buffer := gopacket.NewSerializeBufferExpectedSize(headerSize, 0) + err := gopacket.SerializeLayers(buffer, options, + ip.(gopacket.SerializableLayer), + &tcp, + ) + if err != nil { + return nil, err + } + packetHeaders := buffer.Bytes() + if headerSize != len(packetHeaders) { + return nil, errors.New(fmt.Sprintf("Header check failed %d <> %d", headerSize, len(packetHeaders))) + } + } + + if headerSize+n <= mtu { + buffer := gopacket.NewSerializeBufferExpectedSize(headerSize+n, 0) + err := gopacket.SerializeLayers(buffer, options, + ip.(gopacket.SerializableLayer), + &tcp, + gopacket.Payload(payload[0:n]), + ) + if err != nil { + return nil, err + } + packet := buffer.Bytes() + return [][]byte{packet}, nil + } else { + // fragment + buffer := gopacket.NewSerializeBufferExpectedSize(mtu, 0) + packetSize := mtu - headerSize + packets := [][]byte{} + for i := 0; i < n; { + j := min(i+packetSize, n) + tcp.Seq = self.receiveSeq + uint32(i) + err := gopacket.SerializeLayers(buffer, options, + ip.(gopacket.SerializableLayer), + &tcp, + gopacket.Payload(payload[i:j]), + ) + if err != nil { + return nil, err + } + packet := buffer.Bytes() + packetCopy := make([]byte, len(packet)) + copy(packetCopy, packet) + packets = append(packets, packetCopy) + buffer.Clear() + i = j + } + return packets, nil + } } - func tcpFlagsString(tcp *layers.TCP) string { - flags := []string{} - if tcp.FIN { - flags = append(flags, "FIN") - } - if tcp.SYN { - flags = append(flags, "SYN") - } - if tcp.RST { - flags = append(flags, "RST") - } - if tcp.PSH { - flags = append(flags, "PSH") - } - if tcp.ACK { - flags = append(flags, "ACK") - } - if tcp.URG { - flags = append(flags, "URG") - } - if tcp.ECE { - flags = append(flags, "ECE") - } - if tcp.CWR { - flags = append(flags, "CWR") - } - if tcp.NS { - flags = append(flags, "NS") - } - return strings.Join(flags, ", ") + flags := []string{} + if tcp.FIN { + flags = append(flags, "FIN") + } + if tcp.SYN { + flags = append(flags, "SYN") + } + if tcp.RST { + flags = append(flags, "RST") + } + if tcp.PSH { + flags = append(flags, "PSH") + } + if tcp.ACK { + flags = append(flags, "ACK") + } + if tcp.URG { + flags = append(flags, "URG") + } + if tcp.ECE { + flags = append(flags, "ECE") + } + if tcp.CWR { + flags = append(flags, "CWR") + } + if tcp.NS { + flags = append(flags, "NS") + } + return strings.Join(flags, ", ") } - func DefaultRemoteUserNatProviderSettings() *RemoteUserNatProviderSettings { - return &RemoteUserNatProviderSettings{ - WriteTimeout: 30 * time.Second, - } + return &RemoteUserNatProviderSettings{ + WriteTimeout: 30 * time.Second, + } } - type RemoteUserNatProviderSettings struct { - WriteTimeout time.Duration + WriteTimeout time.Duration } - type RemoteUserNatProvider struct { - client *Client - localUserNat *LocalUserNat - securityPolicy *SecurityPolicy - settings *RemoteUserNatProviderSettings - localUserNatUnsub func() - clientUnsub func() + client *Client + localUserNat *LocalUserNat + securityPolicy *SecurityPolicy + settings *RemoteUserNatProviderSettings + localUserNatUnsub func() + clientUnsub func() } func NewRemoteUserNatProviderWithDefaults( - client *Client, - localUserNat *LocalUserNat, + client *Client, + localUserNat *LocalUserNat, ) *RemoteUserNatProvider { - return NewRemoteUserNatProvider(client, localUserNat, DefaultRemoteUserNatProviderSettings()) + return NewRemoteUserNatProvider(client, localUserNat, DefaultRemoteUserNatProviderSettings()) } func NewRemoteUserNatProvider( - client *Client, - localUserNat *LocalUserNat, - settings *RemoteUserNatProviderSettings, + client *Client, + localUserNat *LocalUserNat, + settings *RemoteUserNatProviderSettings, ) *RemoteUserNatProvider { - userNatProvider := &RemoteUserNatProvider{ - client: client, - localUserNat: localUserNat, - securityPolicy: DefaultSecurityPolicy(), - settings: settings, - } + userNatProvider := &RemoteUserNatProvider{ + client: client, + localUserNat: localUserNat, + securityPolicy: DefaultSecurityPolicy(), + settings: settings, + } - localUserNatUnsub := localUserNat.AddReceivePacketCallback(userNatProvider.Receive) - userNatProvider.localUserNatUnsub = localUserNatUnsub - clientUnsub := client.AddReceiveCallback(userNatProvider.ClientReceive) - userNatProvider.clientUnsub = clientUnsub + localUserNatUnsub := localUserNat.AddReceivePacketCallback(userNatProvider.Receive) + userNatProvider.localUserNatUnsub = localUserNatUnsub + clientUnsub := client.AddReceiveCallback(userNatProvider.ClientReceive) + userNatProvider.clientUnsub = clientUnsub - return userNatProvider + return userNatProvider } // `ReceivePacketFunction` func (self *RemoteUserNatProvider) Receive(source TransferPath, ipProtocol IpProtocol, packet []byte) { - if self.client.ClientId() == source.SourceId { - // locally generated traffic should use a separate local user nat - glog.V(2).Infof("drop remote user nat provider s packet ->%s\n", source.SourceId) - return - } - - ipPacketFromProvider := &protocol.IpPacketFromProvider{ - IpPacket: &protocol.IpPacket{ - PacketBytes: packet, - }, - } - frame, err := ToFrame(ipPacketFromProvider) - if err != nil { - glog.V(2).Infof("drop remote user nat provider s packet ->%s = %s\n", source.SourceId, err) - panic(err) - } - - opts := []any{ - CompanionContract(), - } - switch ipProtocol { - case IpProtocolUdp: - opts = append(opts, NoAck()) - c := func()(bool) { - // ack := make(chan error) - sent := self.client.SendWithTimeout( - frame, - source.Reverse(), - func(err error) {}, - self.settings.WriteTimeout, - opts..., - ) - return sent - } - if glog.V(2) { - TraceWithReturn( - fmt.Sprintf("[unps]udp %s->%s s(%s)", self.client.ClientTag(), source.SourceId, source.StreamId), - c, - ) - } else { - c() - } - case IpProtocolTcp: - c := func()(bool) { - return self.client.SendWithTimeout( - frame, - source.Reverse(), - func(err error) {}, - self.settings.WriteTimeout, - opts..., - ) - } - if glog.V(2) { - TraceWithReturn( - fmt.Sprintf("[unps]tcp %s->%s s(%s)", self.client.ClientTag(), source.SourceId, source.StreamId), - c, - ) - } else { - c() - } - } + if self.client.ClientId() == source.SourceId { + // locally generated traffic should use a separate local user nat + glog.V(2).Infof("drop remote user nat provider s packet ->%s\n", source.SourceId) + return + } + + ipPacketFromProvider := &protocol.IpPacketFromProvider{ + IpPacket: &protocol.IpPacket{ + PacketBytes: packet, + }, + } + frame, err := ToFrame(ipPacketFromProvider) + if err != nil { + glog.V(2).Infof("drop remote user nat provider s packet ->%s = %s\n", source.SourceId, err) + panic(err) + } + + opts := []any{ + CompanionContract(), + } + switch ipProtocol { + case IpProtocolUdp: + opts = append(opts, NoAck()) + c := func() bool { + // ack := make(chan error) + sent := self.client.SendWithTimeout( + frame, + source.Reverse(), + func(err error) {}, + self.settings.WriteTimeout, + opts..., + ) + return sent + } + if glog.V(2) { + TraceWithReturn( + fmt.Sprintf("[unps]udp %s->%s s(%s)", self.client.ClientTag(), source.SourceId, source.StreamId), + c, + ) + } else { + c() + } + case IpProtocolTcp: + c := func() bool { + return self.client.SendWithTimeout( + frame, + source.Reverse(), + func(err error) {}, + self.settings.WriteTimeout, + opts..., + ) + } + if glog.V(2) { + TraceWithReturn( + fmt.Sprintf("[unps]tcp %s->%s s(%s)", self.client.ClientTag(), source.SourceId, source.StreamId), + c, + ) + } else { + c() + } + } } // `connect.ReceiveFunction` func (self *RemoteUserNatProvider) ClientReceive(source TransferPath, frames []*protocol.Frame, provideMode protocol.ProvideMode) { - for _, frame := range frames { - switch frame.MessageType { - case protocol.MessageType_IpIpPacketToProvider: - ipPacketToProvider_, err := FromFrame(frame) - if err != nil { - panic(err) - } - ipPacketToProvider := ipPacketToProvider_.(*protocol.IpPacketToProvider) - - packet := ipPacketToProvider.IpPacket.PacketBytes - _, r := self.securityPolicy.Inspect(provideMode, packet) - switch r { - case SecurityPolicyResultAllow: - c := func()(bool) { - return self.localUserNat.SendPacketWithTimeout(source, provideMode, packet, self.settings.WriteTimeout) - } - if glog.V(2) { - TraceWithReturn( - fmt.Sprintf("[unpr] %s<-%s s(%s)", self.client.ClientTag(), source.SourceId, source.StreamId), - c, - ) - } else { - c() - } - case SecurityPolicyResultIncident: - self.client.ReportAbuse(source) - } - } - } + for _, frame := range frames { + switch frame.MessageType { + case protocol.MessageType_IpIpPacketToProvider: + ipPacketToProvider_, err := FromFrame(frame) + if err != nil { + panic(err) + } + ipPacketToProvider := ipPacketToProvider_.(*protocol.IpPacketToProvider) + + packet := ipPacketToProvider.IpPacket.PacketBytes + _, r := self.securityPolicy.Inspect(provideMode, packet) + switch r { + case SecurityPolicyResultAllow: + c := func() bool { + return self.localUserNat.SendPacketWithTimeout(source, provideMode, packet, self.settings.WriteTimeout) + } + if glog.V(2) { + TraceWithReturn( + fmt.Sprintf("[unpr] %s<-%s s(%s)", self.client.ClientTag(), source.SourceId, source.StreamId), + c, + ) + } else { + c() + } + case SecurityPolicyResultIncident: + self.client.ReportAbuse(source) + } + } + } } func (self *RemoteUserNatProvider) Close() { - // self.client.RemoveReceiveCallback(self.clientCallbackId) - // self.localUserNat.RemoveReceivePacketCallback(self.localUserNatCallbackId) - self.clientUnsub() - self.localUserNatUnsub() + // self.client.RemoveReceiveCallback(self.clientCallbackId) + // self.localUserNat.RemoveReceivePacketCallback(self.localUserNatCallbackId) + self.clientUnsub() + self.localUserNatUnsub() } - // this is a basic implementation. See `RemoteUserNatWindowedClient` for a more robust implementation type RemoteUserNatClient struct { - client *Client - receivePacketCallback ReceivePacketFunction - securityPolicy *SecurityPolicy - pathTable *pathTable - // the provide mode of the source packets - // for locally generated packets this is `ProvideMode_Network` - provideMode protocol.ProvideMode - clientUnsub func() + client *Client + receivePacketCallback ReceivePacketFunction + securityPolicy *SecurityPolicy + pathTable *pathTable + // the provide mode of the source packets + // for locally generated packets this is `ProvideMode_Network` + provideMode protocol.ProvideMode + clientUnsub func() } func NewRemoteUserNatClient( - client *Client, - receivePacketCallback ReceivePacketFunction, - destinations []MultiHopId, - provideMode protocol.ProvideMode, + client *Client, + receivePacketCallback ReceivePacketFunction, + destinations []MultiHopId, + provideMode protocol.ProvideMode, ) (*RemoteUserNatClient, error) { - pathTable, err := newPathTable(destinations) - if err != nil { - return nil, err - } + pathTable, err := newPathTable(destinations) + if err != nil { + return nil, err + } - userNatClient := &RemoteUserNatClient{ - client: client, - receivePacketCallback: receivePacketCallback, - securityPolicy: DefaultSecurityPolicy(), - pathTable: pathTable, - provideMode: provideMode, - } + userNatClient := &RemoteUserNatClient{ + client: client, + receivePacketCallback: receivePacketCallback, + securityPolicy: DefaultSecurityPolicy(), + pathTable: pathTable, + provideMode: provideMode, + } - clientUnsub := client.AddReceiveCallback(userNatClient.ClientReceive) - userNatClient.clientUnsub = clientUnsub + clientUnsub := client.AddReceiveCallback(userNatClient.ClientReceive) + userNatClient.clientUnsub = clientUnsub - return userNatClient, nil + return userNatClient, nil } // `SendPacketFunction` func (self *RemoteUserNatClient) SendPacket(source TransferPath, provideMode protocol.ProvideMode, packet []byte, timeout time.Duration) bool { - minRelationship := max(provideMode, self.provideMode) - - ipPath, r := self.securityPolicy.Inspect(minRelationship, packet) - switch r { - case SecurityPolicyResultAllow: - destination, err := self.pathTable.SelectDestination(packet) - if err != nil { - // drop - return false - } - - ipPacketToProvider := &protocol.IpPacketToProvider{ - IpPacket: &protocol.IpPacket{ - PacketBytes: packet, - }, - } - frame, err := ToFrame(ipPacketToProvider) - if err != nil { - panic(err) - } - - // the sender will control transfer - opts := []any{} - switch ipPath.Protocol { - case IpProtocolUdp: - opts = append(opts, NoAck()) - } - success := self.client.SendMultiHopWithTimeout(frame, destination, func(err error) {}, timeout, opts...) - return success - default: - return false - } + minRelationship := max(provideMode, self.provideMode) + + ipPath, r := self.securityPolicy.Inspect(minRelationship, packet) + switch r { + case SecurityPolicyResultAllow: + destination, err := self.pathTable.SelectDestination(packet) + if err != nil { + // drop + return false + } + + ipPacketToProvider := &protocol.IpPacketToProvider{ + IpPacket: &protocol.IpPacket{ + PacketBytes: packet, + }, + } + frame, err := ToFrame(ipPacketToProvider) + if err != nil { + panic(err) + } + + // the sender will control transfer + opts := []any{} + switch ipPath.Protocol { + case IpProtocolUdp: + opts = append(opts, NoAck()) + } + success := self.client.SendMultiHopWithTimeout(frame, destination, func(err error) {}, timeout, opts...) + return success + default: + return false + } } // `connect.ReceiveFunction` func (self *RemoteUserNatClient) ClientReceive(source TransferPath, frames []*protocol.Frame, provideMode protocol.ProvideMode) { - // only process frames from the destinations - // if allow := self.sourceFilter[source]; !allow { - // return - // } - - for _, frame := range frames { - switch frame.MessageType { - case protocol.MessageType_IpIpPacketFromProvider: - ipPacketFromProvider_, err := FromFrame(frame) - if err != nil { - panic(err) - } - ipPacketFromProvider := ipPacketFromProvider_.(*protocol.IpPacketFromProvider) - - HandleError(func() { - self.receivePacketCallback(source, IpProtocolUnknown, ipPacketFromProvider.IpPacket.PacketBytes) - }) - } - } + // only process frames from the destinations + // if allow := self.sourceFilter[source]; !allow { + // return + // } + + for _, frame := range frames { + switch frame.MessageType { + case protocol.MessageType_IpIpPacketFromProvider: + ipPacketFromProvider_, err := FromFrame(frame) + if err != nil { + panic(err) + } + ipPacketFromProvider := ipPacketFromProvider_.(*protocol.IpPacketFromProvider) + + HandleError(func() { + self.receivePacketCallback(source, IpProtocolUnknown, ipPacketFromProvider.IpPacket.PacketBytes) + }) + } + } } func (self *RemoteUserNatClient) Shuffle() { } func (self *RemoteUserNatClient) Close() { - // self.client.RemoveReceiveCallback(self.clientCallbackId) - self.clientUnsub() + // self.client.RemoveReceiveCallback(self.clientCallbackId) + self.clientUnsub() } - type pathTable struct { - destinations []MultiHopId + destinations []MultiHopId - // TODO clean up entries that haven't been used in some time - paths4 map[Ip4Path]MultiHopId - paths6 map[Ip6Path]MultiHopId + // TODO clean up entries that haven't been used in some time + paths4 map[Ip4Path]MultiHopId + paths6 map[Ip6Path]MultiHopId } func newPathTable(destinations []MultiHopId) (*pathTable, error) { - if len(destinations) == 0 { - return nil, errors.New("No destinations.") - } - return &pathTable{ - destinations: destinations, - paths4: map[Ip4Path]MultiHopId{}, - paths6: map[Ip6Path]MultiHopId{}, - }, nil + if len(destinations) == 0 { + return nil, errors.New("No destinations.") + } + return &pathTable{ + destinations: destinations, + paths4: map[Ip4Path]MultiHopId{}, + paths6: map[Ip6Path]MultiHopId{}, + }, nil } func (self *pathTable) SelectDestination(packet []byte) (MultiHopId, error) { - if len(self.destinations) == 1 { - return self.destinations[0], nil - } - - ipPath, err := ParseIpPath(packet) - if err != nil { - return MultiHopId{}, err - } - switch ipPath.Version { - case 4: - ip4Path := ipPath.ToIp4Path() - if destination, ok := self.paths4[ip4Path]; ok { - return destination, nil - } - i := mathrand.Intn(len(self.destinations)) - destination := self.destinations[i] - self.paths4[ip4Path] = destination - return destination, nil - case 6: - ip6Path := ipPath.ToIp6Path() - if destination, ok := self.paths6[ip6Path]; ok { - return destination, nil - } - i := mathrand.Intn(len(self.destinations)) - destination := self.destinations[i] - self.paths6[ip6Path] = destination - return destination, nil - default: - // no support for this version - return MultiHopId{}, fmt.Errorf("No support for ip version %d", ipPath.Version) - } + if len(self.destinations) == 1 { + return self.destinations[0], nil + } + + ipPath, err := ParseIpPath(packet) + if err != nil { + return MultiHopId{}, err + } + switch ipPath.Version { + case 4: + ip4Path := ipPath.ToIp4Path() + if destination, ok := self.paths4[ip4Path]; ok { + return destination, nil + } + i := mathrand.Intn(len(self.destinations)) + destination := self.destinations[i] + self.paths4[ip4Path] = destination + return destination, nil + case 6: + ip6Path := ipPath.ToIp6Path() + if destination, ok := self.paths6[ip6Path]; ok { + return destination, nil + } + i := mathrand.Intn(len(self.destinations)) + destination := self.destinations[i] + self.paths6[ip6Path] = destination + return destination, nil + default: + // no support for this version + return MultiHopId{}, fmt.Errorf("No support for ip version %d", ipPath.Version) + } } - type IpProtocol int + const ( - IpProtocolUnknown IpProtocol = 0 - IpProtocolTcp IpProtocol = 1 - IpProtocolUdp IpProtocol = 2 + IpProtocolUnknown IpProtocol = 0 + IpProtocolTcp IpProtocol = 1 + IpProtocolUdp IpProtocol = 2 ) - type IpPath struct { - Version int - Protocol IpProtocol - SourceIp net.IP - SourcePort int - DestinationIp net.IP - DestinationPort int + Version int + Protocol IpProtocol + SourceIp net.IP + SourcePort int + DestinationIp net.IP + DestinationPort int } func ParseIpPath(ipPacket []byte) (*IpPath, error) { - ipVersion := uint8(ipPacket[0]) >> 4 - switch ipVersion { - case 4: - ipv4 := layers.IPv4{} - ipv4.DecodeFromBytes(ipPacket, gopacket.NilDecodeFeedback) - switch ipv4.Protocol { - case layers.IPProtocolUDP: - udp := layers.UDP{} - udp.DecodeFromBytes(ipv4.Payload, gopacket.NilDecodeFeedback) - - return &IpPath { - Version: int(ipVersion), - Protocol: IpProtocolUdp, - SourceIp: ipv4.SrcIP, - SourcePort: int(udp.SrcPort), - DestinationIp: ipv4.DstIP, - DestinationPort: int(udp.DstPort), - }, nil - case layers.IPProtocolTCP: - tcp := layers.TCP{} - tcp.DecodeFromBytes(ipv4.Payload, gopacket.NilDecodeFeedback) - - return &IpPath { - Version: int(ipVersion), - Protocol: IpProtocolTcp, - SourceIp: ipv4.SrcIP, - SourcePort: int(tcp.SrcPort), - DestinationIp: ipv4.DstIP, - DestinationPort: int(tcp.DstPort), - }, nil - default: - // no support for this protocol - return nil, fmt.Errorf("No support for protocol %d", ipv4.Protocol) - } - case 6: - ipv6 := layers.IPv6{} - ipv6.DecodeFromBytes(ipPacket, gopacket.NilDecodeFeedback) - switch ipv6.NextHeader { - case layers.IPProtocolUDP: - udp := layers.UDP{} - udp.DecodeFromBytes(ipv6.Payload, gopacket.NilDecodeFeedback) - - return &IpPath { - Version: int(ipVersion), - Protocol: IpProtocolUdp, - SourceIp: ipv6.SrcIP, - SourcePort: int(udp.SrcPort), - DestinationIp: ipv6.DstIP, - DestinationPort: int(udp.DstPort), - }, nil - case layers.IPProtocolTCP: - tcp := layers.TCP{} - tcp.DecodeFromBytes(ipv6.Payload, gopacket.NilDecodeFeedback) - - return &IpPath { - Version: int(ipVersion), - Protocol: IpProtocolTcp, - SourceIp: ipv6.SrcIP, - SourcePort: int(tcp.SrcPort), - DestinationIp: ipv6.DstIP, - DestinationPort: int(tcp.DstPort), - }, nil - default: - // no support for this protocol - return nil, fmt.Errorf("No support for protocol %d", ipv6.NextHeader) - } - default: - // no support for this version - return nil, fmt.Errorf("No support for ip version %d", ipVersion) - } + ipVersion := uint8(ipPacket[0]) >> 4 + switch ipVersion { + case 4: + ipv4 := layers.IPv4{} + ipv4.DecodeFromBytes(ipPacket, gopacket.NilDecodeFeedback) + switch ipv4.Protocol { + case layers.IPProtocolUDP: + udp := layers.UDP{} + udp.DecodeFromBytes(ipv4.Payload, gopacket.NilDecodeFeedback) + + return &IpPath{ + Version: int(ipVersion), + Protocol: IpProtocolUdp, + SourceIp: ipv4.SrcIP, + SourcePort: int(udp.SrcPort), + DestinationIp: ipv4.DstIP, + DestinationPort: int(udp.DstPort), + }, nil + case layers.IPProtocolTCP: + tcp := layers.TCP{} + tcp.DecodeFromBytes(ipv4.Payload, gopacket.NilDecodeFeedback) + + return &IpPath{ + Version: int(ipVersion), + Protocol: IpProtocolTcp, + SourceIp: ipv4.SrcIP, + SourcePort: int(tcp.SrcPort), + DestinationIp: ipv4.DstIP, + DestinationPort: int(tcp.DstPort), + }, nil + default: + // no support for this protocol + return nil, fmt.Errorf("No support for protocol %d", ipv4.Protocol) + } + case 6: + ipv6 := layers.IPv6{} + ipv6.DecodeFromBytes(ipPacket, gopacket.NilDecodeFeedback) + switch ipv6.NextHeader { + case layers.IPProtocolUDP: + udp := layers.UDP{} + udp.DecodeFromBytes(ipv6.Payload, gopacket.NilDecodeFeedback) + + return &IpPath{ + Version: int(ipVersion), + Protocol: IpProtocolUdp, + SourceIp: ipv6.SrcIP, + SourcePort: int(udp.SrcPort), + DestinationIp: ipv6.DstIP, + DestinationPort: int(udp.DstPort), + }, nil + case layers.IPProtocolTCP: + tcp := layers.TCP{} + tcp.DecodeFromBytes(ipv6.Payload, gopacket.NilDecodeFeedback) + + return &IpPath{ + Version: int(ipVersion), + Protocol: IpProtocolTcp, + SourceIp: ipv6.SrcIP, + SourcePort: int(tcp.SrcPort), + DestinationIp: ipv6.DstIP, + DestinationPort: int(tcp.DstPort), + }, nil + default: + // no support for this protocol + return nil, fmt.Errorf("No support for protocol %d", ipv6.NextHeader) + } + default: + // no support for this version + return nil, fmt.Errorf("No support for ip version %d", ipVersion) + } } func (self *IpPath) ToIp4Path() Ip4Path { - var sourceIp [4]byte - if self.SourceIp != nil { - if sourceIp4 := self.SourceIp.To4(); sourceIp4 != nil { - sourceIp = [4]byte(sourceIp4) - } - } - var destinationIp [4]byte - if self.DestinationIp != nil { - if destinationIp4 := self.DestinationIp.To4(); destinationIp4 != nil { - destinationIp = [4]byte(destinationIp4) - } - } - return Ip4Path{ - Protocol: self.Protocol, - SourceIp: sourceIp, - SourcePort: self.SourcePort, - DestinationIp: destinationIp, - DestinationPort: self.DestinationPort, - } + var sourceIp [4]byte + if self.SourceIp != nil { + if sourceIp4 := self.SourceIp.To4(); sourceIp4 != nil { + sourceIp = [4]byte(sourceIp4) + } + } + var destinationIp [4]byte + if self.DestinationIp != nil { + if destinationIp4 := self.DestinationIp.To4(); destinationIp4 != nil { + destinationIp = [4]byte(destinationIp4) + } + } + return Ip4Path{ + Protocol: self.Protocol, + SourceIp: sourceIp, + SourcePort: self.SourcePort, + DestinationIp: destinationIp, + DestinationPort: self.DestinationPort, + } } func (self *IpPath) ToIp6Path() Ip6Path { - var sourceIp [16]byte - if self.SourceIp != nil { - if sourceIp6 := self.SourceIp.To16(); sourceIp6 != nil { - sourceIp = [16]byte(sourceIp6) - } - } - var destinationIp [16]byte - if self.DestinationIp != nil { - if destinationIp6 := self.DestinationIp.To16(); destinationIp6 != nil { - destinationIp = [16]byte(destinationIp6) - } - } - return Ip6Path{ - Protocol: self.Protocol, - SourceIp: sourceIp, - SourcePort: self.SourcePort, - DestinationIp: destinationIp, - DestinationPort: self.DestinationPort, - } + var sourceIp [16]byte + if self.SourceIp != nil { + if sourceIp6 := self.SourceIp.To16(); sourceIp6 != nil { + sourceIp = [16]byte(sourceIp6) + } + } + var destinationIp [16]byte + if self.DestinationIp != nil { + if destinationIp6 := self.DestinationIp.To16(); destinationIp6 != nil { + destinationIp = [16]byte(destinationIp6) + } + } + return Ip6Path{ + Protocol: self.Protocol, + SourceIp: sourceIp, + SourcePort: self.SourcePort, + DestinationIp: destinationIp, + DestinationPort: self.DestinationPort, + } } func (self *IpPath) Source() *IpPath { - return &IpPath{ - Protocol: self.Protocol, - Version: self.Version, - SourceIp: self.SourceIp, - SourcePort: self.SourcePort, - } + return &IpPath{ + Protocol: self.Protocol, + Version: self.Version, + SourceIp: self.SourceIp, + SourcePort: self.SourcePort, + } } func (self *IpPath) Destination() *IpPath { - return &IpPath{ - Protocol: self.Protocol, - Version: self.Version, - DestinationIp: self.DestinationIp, - DestinationPort: self.DestinationPort, - } + return &IpPath{ + Protocol: self.Protocol, + Version: self.Version, + DestinationIp: self.DestinationIp, + DestinationPort: self.DestinationPort, + } } - // comparable type Ip4Path struct { - Protocol IpProtocol - SourceIp [4]byte - SourcePort int - DestinationIp [4]byte - DestinationPort int + Protocol IpProtocol + SourceIp [4]byte + SourcePort int + DestinationIp [4]byte + DestinationPort int } func (self *Ip4Path) Source() Ip4Path { - return Ip4Path{ - Protocol: self.Protocol, - SourceIp: self.SourceIp, - SourcePort: self.SourcePort, - } + return Ip4Path{ + Protocol: self.Protocol, + SourceIp: self.SourceIp, + SourcePort: self.SourcePort, + } } func (self *Ip4Path) Destination() Ip4Path { - return Ip4Path{ - Protocol: self.Protocol, - DestinationIp: self.DestinationIp, - DestinationPort: self.DestinationPort, - } + return Ip4Path{ + Protocol: self.Protocol, + DestinationIp: self.DestinationIp, + DestinationPort: self.DestinationPort, + } } - // comparable type Ip6Path struct { - Protocol IpProtocol - SourceIp [16]byte - SourcePort int - DestinationIp [16]byte - DestinationPort int + Protocol IpProtocol + SourceIp [16]byte + SourcePort int + DestinationIp [16]byte + DestinationPort int } func (self *Ip6Path) Source() Ip6Path { - return Ip6Path{ - Protocol: self.Protocol, - SourceIp: self.SourceIp, - SourcePort: self.SourcePort, - } + return Ip6Path{ + Protocol: self.Protocol, + SourceIp: self.SourceIp, + SourcePort: self.SourcePort, + } } func (self *Ip6Path) Destination() Ip6Path { - return Ip6Path{ - Protocol: self.Protocol, - DestinationIp: self.DestinationIp, - DestinationPort: self.DestinationPort, - } + return Ip6Path{ + Protocol: self.Protocol, + DestinationIp: self.DestinationIp, + DestinationPort: self.DestinationPort, + } } - type SecurityPolicyResult int + const ( - SecurityPolicyResultDrop SecurityPolicyResult = 0 - SecurityPolicyResultAllow SecurityPolicyResult = 1 - SecurityPolicyResultIncident SecurityPolicyResult = 2 + SecurityPolicyResultDrop SecurityPolicyResult = 0 + SecurityPolicyResultAllow SecurityPolicyResult = 1 + SecurityPolicyResultIncident SecurityPolicyResult = 2 ) - type SecurityPolicy struct { } func DefaultSecurityPolicy() *SecurityPolicy { - return &SecurityPolicy{} + return &SecurityPolicy{} } func (self *SecurityPolicy) Inspect(provideMode protocol.ProvideMode, packet []byte) (*IpPath, SecurityPolicyResult) { - ipPath, err := ParseIpPath(packet) - if err != nil { - // back ip packet - return ipPath, SecurityPolicyResultDrop - } - - if protocol.ProvideMode_Public <= provideMode { - // apply public rules: - // - only public unicast network destinations - // - block insecure or known unencrypted traffic - - if !isPublicUnicast(ipPath.DestinationIp) { - return ipPath, SecurityPolicyResultIncident - } - - // block insecure or unencrypted traffic is implemented as a block list, - // rather than an allow list. - // Known insecure traffic and unencrypted is blocked. - // This currently includes: - // - port 80 (http) - // - ports 6881 to 6889 (bittorrent) - allow := func()(bool) { - switch port := ipPath.DestinationPort; { - case port == 80: - // http - return false - case 6881 <= port && port <= 6889: - // bittorrent - return false - default: - return true - } - } - if !allow() { - return ipPath, SecurityPolicyResultDrop - } - } - - return ipPath, SecurityPolicyResultAllow + ipPath, err := ParseIpPath(packet) + if err != nil { + // back ip packet + return ipPath, SecurityPolicyResultDrop + } + + if protocol.ProvideMode_Public <= provideMode { + // apply public rules: + // - only public unicast network destinations + // - block insecure or known unencrypted traffic + + if !isPublicUnicast(ipPath.DestinationIp) { + return ipPath, SecurityPolicyResultIncident + } + + // block insecure or unencrypted traffic is implemented as a block list, + // rather than an allow list. + // Known insecure traffic and unencrypted is blocked. + // This currently includes: + // - port 80 (http) + // - ports 6881 to 6889 (bittorrent) + allow := func() bool { + switch port := ipPath.DestinationPort; { + case port == 80: + // http + return false + case 6881 <= port && port <= 6889: + // bittorrent + return false + default: + return true + } + } + if !allow() { + return ipPath, SecurityPolicyResultDrop + } + } + + return ipPath, SecurityPolicyResultAllow } - func isPublicUnicast(ip net.IP) bool { - switch { - case ip.IsPrivate(), - ip.IsLoopback(), - ip.IsLinkLocalUnicast(), - ip.IsMulticast(), - ip.IsUnspecified(): - return false - default: - return true - } + switch { + case ip.IsPrivate(), + ip.IsLoopback(), + ip.IsLinkLocalUnicast(), + ip.IsMulticast(), + ip.IsUnspecified(): + return false + default: + return true + } } - type UserLimited interface { - LastActivityTime() time.Time - Cancel() + LastActivityTime() time.Time + Cancel() } type userLimited struct { - mutex sync.Mutex - lastActivityTime time.Time + mutex sync.Mutex + lastActivityTime time.Time } func newUserLimited() *userLimited { - return &userLimited{ - lastActivityTime: time.Now(), - } + return &userLimited{ + lastActivityTime: time.Now(), + } } func (self *userLimited) LastActivityTime() time.Time { - self.mutex.Lock() - defer self.mutex.Unlock() - return self.lastActivityTime + self.mutex.Lock() + defer self.mutex.Unlock() + return self.lastActivityTime } func (self *userLimited) UpdateLastActivityTime() { - self.mutex.Lock() - defer self.mutex.Unlock() - self.lastActivityTime = time.Now() -} - - -func applyLruUserLimit[R UserLimited](resources []R, ulimit int, limitCallback func(R)(bool)) { - // limit the total connections per source to avoid blowing up the ulimit - if n := len(resources) - ulimit; 0 < n { - resourceLastActivityTimes := map[UserLimited]time.Time{} - for _, resource := range resources { - resourceLastActivityTimes[resource] = resource.LastActivityTime() - } - // order by last activity time - slices.SortFunc(resources, func(a R, b R)(int) { - lastActivityTimeA := resourceLastActivityTimes[a] - lastActivityTimeB := resourceLastActivityTimes[b] - if lastActivityTimeA.Before(lastActivityTimeB) { - return -1 - } else if lastActivityTimeB.Before(lastActivityTimeA) { - return 1 - } else { - return 0 - } - }) - i := 0 - for _, resource := range resources { - if limitCallback(resource) { - i += 1 - resource.Cancel() - } - if n <= i { - break - } - } - } + self.mutex.Lock() + defer self.mutex.Unlock() + self.lastActivityTime = time.Now() +} + +func applyLruUserLimit[R UserLimited](resources []R, ulimit int, limitCallback func(R) bool) { + // limit the total connections per source to avoid blowing up the ulimit + if n := len(resources) - ulimit; 0 < n { + resourceLastActivityTimes := map[UserLimited]time.Time{} + for _, resource := range resources { + resourceLastActivityTimes[resource] = resource.LastActivityTime() + } + // order by last activity time + slices.SortFunc(resources, func(a R, b R) int { + lastActivityTimeA := resourceLastActivityTimes[a] + lastActivityTimeB := resourceLastActivityTimes[b] + if lastActivityTimeA.Before(lastActivityTimeB) { + return -1 + } else if lastActivityTimeB.Before(lastActivityTimeA) { + return 1 + } else { + return 0 + } + }) + i := 0 + for _, resource := range resources { + if limitCallback(resource) { + i += 1 + resource.Cancel() + } + if n <= i { + break + } + } + } } - - - - diff --git a/connect/ip_remote_multi_client.go b/connect/ip_remote_multi_client.go index c385338..bae5f31 100644 --- a/connect/ip_remote_multi_client.go +++ b/connect/ip_remote_multi_client.go @@ -1,26 +1,26 @@ package connect import ( - "context" - "time" - "sync" - // "reflect" - "errors" - "fmt" - "slices" - "math" - mathrand "math/rand" - "strings" + "context" + "sync" + "time" - "golang.org/x/exp/maps" + // "reflect" + "errors" + "fmt" + "math" + mathrand "math/rand" + "slices" + "strings" - "google.golang.org/protobuf/proto" + "golang.org/x/exp/maps" - "github.com/golang/glog" + "google.golang.org/protobuf/proto" - "bringyour.com/protocol" -) + "github.com/golang/glog" + "github.com/bringyour/connect/protocol" +) // multi client is a sender approach to mitigate bad destinations // it maintains a window of compatible clients chosen using specs @@ -28,1778 +28,1758 @@ import ( // - the clients are rate limited by the number of outstanding acks (nacks) // - the size of allowed outstanding nacks increases with each ack, // scaling up successful destinations to use the full transfer buffer -// - the clients are chosen with probability weighted by their +// - the clients are chosen with probability weighted by their // net frame count statistics (acks - nacks) - // TODO surface window stats to show to users - -// for each `NewClientArgs`, -// `RemoveClientWithArgs` will be called if a client was created for the args, -// else `RemoveClientArgs` +// for each `NewClientArgs`, +// +// `RemoveClientWithArgs` will be called if a client was created for the args, +// else `RemoveClientArgs` type MultiClientGenerator interface { - // path -> estimated byte count per second - // the enumeration should typically - // 1. not repeat final destination ids from any path - // 2. not repeat intermediary elements from any path - NextDestinations(count int, excludeDestinations []MultiHopId) (map[MultiHopId]ByteCount, error) - // client id, client auth - NewClientArgs() (*MultiClientGeneratorClientArgs, error) - RemoveClientArgs(args *MultiClientGeneratorClientArgs) - RemoveClientWithArgs(client *Client, args *MultiClientGeneratorClientArgs) - NewClientSettings() *ClientSettings - NewClient(ctx context.Context, args *MultiClientGeneratorClientArgs, clientSettings *ClientSettings) (*Client, error) + // path -> estimated byte count per second + // the enumeration should typically + // 1. not repeat final destination ids from any path + // 2. not repeat intermediary elements from any path + NextDestinations(count int, excludeDestinations []MultiHopId) (map[MultiHopId]ByteCount, error) + // client id, client auth + NewClientArgs() (*MultiClientGeneratorClientArgs, error) + RemoveClientArgs(args *MultiClientGeneratorClientArgs) + RemoveClientWithArgs(client *Client, args *MultiClientGeneratorClientArgs) + NewClientSettings() *ClientSettings + NewClient(ctx context.Context, args *MultiClientGeneratorClientArgs, clientSettings *ClientSettings) (*Client, error) } - func DefaultMultiClientSettings() *MultiClientSettings { - return &MultiClientSettings{ - WindowSizeMin: 2, - // TODO increase this when p2p is deployed - WindowSizeMinP2pOnly: 0, - WindowSizeMax: 8, - // reconnects per source - WindowSizeReconnectScale: 1.0, - WriteRetryTimeout: 200 * time.Millisecond, - PingWriteTimeout: 1 * time.Second, - PingTimeout: 5 * time.Second, - // a lower ack timeout helps cycle through bad providers faster - AckTimeout: 10 * time.Second, - BlackholeTimeout: 10 * time.Second, - WindowResizeTimeout: 5 * time.Second, - StatsWindowGraceperiod: 5 * time.Second, - StatsWindowEntropy: 0.25, - WindowExpandTimeout: 15 * time.Second, - // wait this time before enumerating potential clients again - WindowEnumerateEmptyTimeout: 60 * time.Second, - WindowEnumerateErrorTimeout: 1 * time.Second, - WindowExpandScale: 2.0, - WindowCollapseScale: 0.5, - WindowExpandMaxOvershotScale: 4.0, - StatsWindowDuration: 120 * time.Second, - StatsWindowBucketDuration: 10 * time.Second, - StatsSampleWeightsCount: 8, - StatsSourceCountSelection: 0.95, - - RemoteUserNatMultiClientMonitorSettings: *DefaultRemoteUserNatMultiClientMonitorSettings(), - } + return &MultiClientSettings{ + WindowSizeMin: 2, + // TODO increase this when p2p is deployed + WindowSizeMinP2pOnly: 0, + WindowSizeMax: 8, + // reconnects per source + WindowSizeReconnectScale: 1.0, + WriteRetryTimeout: 200 * time.Millisecond, + PingWriteTimeout: 1 * time.Second, + PingTimeout: 5 * time.Second, + // a lower ack timeout helps cycle through bad providers faster + AckTimeout: 10 * time.Second, + BlackholeTimeout: 10 * time.Second, + WindowResizeTimeout: 5 * time.Second, + StatsWindowGraceperiod: 5 * time.Second, + StatsWindowEntropy: 0.25, + WindowExpandTimeout: 15 * time.Second, + // wait this time before enumerating potential clients again + WindowEnumerateEmptyTimeout: 60 * time.Second, + WindowEnumerateErrorTimeout: 1 * time.Second, + WindowExpandScale: 2.0, + WindowCollapseScale: 0.5, + WindowExpandMaxOvershotScale: 4.0, + StatsWindowDuration: 120 * time.Second, + StatsWindowBucketDuration: 10 * time.Second, + StatsSampleWeightsCount: 8, + StatsSourceCountSelection: 0.95, + + RemoteUserNatMultiClientMonitorSettings: *DefaultRemoteUserNatMultiClientMonitorSettings(), + } } - type MultiClientSettings struct { - WindowSizeMin int - // the minimumum number of items in the windows that must be connected via p2p only - WindowSizeMinP2pOnly int - WindowSizeMax int - // reconnects per source - WindowSizeReconnectScale float64 - // ClientNackInitialLimit int - // ClientNackMaxLimit int - // ClientNackScale float64 - // ClientWriteTimeout time.Duration - // SendTimeout time.Duration - // WriteTimeout time.Duration - WriteRetryTimeout time.Duration - PingWriteTimeout time.Duration - PingTimeout time.Duration - AckTimeout time.Duration - BlackholeTimeout time.Duration - WindowResizeTimeout time.Duration - StatsWindowGraceperiod time.Duration - StatsWindowEntropy float32 - WindowExpandTimeout time.Duration - WindowEnumerateEmptyTimeout time.Duration - WindowEnumerateErrorTimeout time.Duration - WindowExpandScale float64 - WindowCollapseScale float64 - WindowExpandMaxOvershotScale float64 - StatsWindowDuration time.Duration - StatsWindowBucketDuration time.Duration - StatsSampleWeightsCount int - StatsSourceCountSelection float64 - - RemoteUserNatMultiClientMonitorSettings + WindowSizeMin int + // the minimumum number of items in the windows that must be connected via p2p only + WindowSizeMinP2pOnly int + WindowSizeMax int + // reconnects per source + WindowSizeReconnectScale float64 + // ClientNackInitialLimit int + // ClientNackMaxLimit int + // ClientNackScale float64 + // ClientWriteTimeout time.Duration + // SendTimeout time.Duration + // WriteTimeout time.Duration + WriteRetryTimeout time.Duration + PingWriteTimeout time.Duration + PingTimeout time.Duration + AckTimeout time.Duration + BlackholeTimeout time.Duration + WindowResizeTimeout time.Duration + StatsWindowGraceperiod time.Duration + StatsWindowEntropy float32 + WindowExpandTimeout time.Duration + WindowEnumerateEmptyTimeout time.Duration + WindowEnumerateErrorTimeout time.Duration + WindowExpandScale float64 + WindowCollapseScale float64 + WindowExpandMaxOvershotScale float64 + StatsWindowDuration time.Duration + StatsWindowBucketDuration time.Duration + StatsSampleWeightsCount int + StatsSourceCountSelection float64 + + RemoteUserNatMultiClientMonitorSettings } - type RemoteUserNatMultiClient struct { - ctx context.Context - cancel context.CancelFunc + ctx context.Context + cancel context.CancelFunc - generator MultiClientGenerator + generator MultiClientGenerator - receivePacketCallback ReceivePacketFunction + receivePacketCallback ReceivePacketFunction - settings *MultiClientSettings + settings *MultiClientSettings - window *multiClientWindow + window *multiClientWindow - stateLock sync.Mutex - ip4PathUpdates map[Ip4Path]*multiClientChannelUpdate - ip6PathUpdates map[Ip6Path]*multiClientChannelUpdate - updateIp4Paths map[*multiClientChannelUpdate]map[Ip4Path]bool - updateIp6Paths map[*multiClientChannelUpdate]map[Ip6Path]bool - clientUpdates map[*multiClientChannel]*multiClientChannelUpdate + stateLock sync.Mutex + ip4PathUpdates map[Ip4Path]*multiClientChannelUpdate + ip6PathUpdates map[Ip6Path]*multiClientChannelUpdate + updateIp4Paths map[*multiClientChannelUpdate]map[Ip4Path]bool + updateIp6Paths map[*multiClientChannelUpdate]map[Ip6Path]bool + clientUpdates map[*multiClientChannel]*multiClientChannelUpdate } func NewRemoteUserNatMultiClientWithDefaults( - ctx context.Context, - generator MultiClientGenerator, - receivePacketCallback ReceivePacketFunction, + ctx context.Context, + generator MultiClientGenerator, + receivePacketCallback ReceivePacketFunction, ) *RemoteUserNatMultiClient { - return NewRemoteUserNatMultiClient( - ctx, - generator, - receivePacketCallback, - DefaultMultiClientSettings(), - ) + return NewRemoteUserNatMultiClient( + ctx, + generator, + receivePacketCallback, + DefaultMultiClientSettings(), + ) } func NewRemoteUserNatMultiClient( - ctx context.Context, - generator MultiClientGenerator, - receivePacketCallback ReceivePacketFunction, - settings *MultiClientSettings, + ctx context.Context, + generator MultiClientGenerator, + receivePacketCallback ReceivePacketFunction, + settings *MultiClientSettings, ) *RemoteUserNatMultiClient { - cancelCtx, cancel := context.WithCancel(ctx) - - window := newMultiClientWindow( - cancelCtx, - cancel, - generator, - receivePacketCallback, - settings, - ) - - return &RemoteUserNatMultiClient{ - ctx: cancelCtx, - cancel: cancel, - generator: generator, - receivePacketCallback: receivePacketCallback, - settings: settings, - window: window, - ip4PathUpdates: map[Ip4Path]*multiClientChannelUpdate{}, - ip6PathUpdates: map[Ip6Path]*multiClientChannelUpdate{}, - updateIp4Paths: map[*multiClientChannelUpdate]map[Ip4Path]bool{}, - updateIp6Paths: map[*multiClientChannelUpdate]map[Ip6Path]bool{}, - clientUpdates: map[*multiClientChannel]*multiClientChannelUpdate{}, - } + cancelCtx, cancel := context.WithCancel(ctx) + + window := newMultiClientWindow( + cancelCtx, + cancel, + generator, + receivePacketCallback, + settings, + ) + + return &RemoteUserNatMultiClient{ + ctx: cancelCtx, + cancel: cancel, + generator: generator, + receivePacketCallback: receivePacketCallback, + settings: settings, + window: window, + ip4PathUpdates: map[Ip4Path]*multiClientChannelUpdate{}, + ip6PathUpdates: map[Ip6Path]*multiClientChannelUpdate{}, + updateIp4Paths: map[*multiClientChannelUpdate]map[Ip4Path]bool{}, + updateIp6Paths: map[*multiClientChannelUpdate]map[Ip6Path]bool{}, + clientUpdates: map[*multiClientChannel]*multiClientChannelUpdate{}, + } } func (self *RemoteUserNatMultiClient) Monitor() *RemoteUserNatMultiClientMonitor { - return self.window.monitor + return self.window.monitor } func (self *RemoteUserNatMultiClient) updateClientPath(ipPath *IpPath, callback func(*multiClientChannelUpdate)) { - reserveUpdate := func()(*multiClientChannelUpdate) { - self.stateLock.Lock() - defer self.stateLock.Unlock() - - switch ipPath.Version { - case 4: - ip4Path := ipPath.ToIp4Path() - update, ok := self.ip4PathUpdates[ip4Path] - if !ok { - update = &multiClientChannelUpdate{} - self.ip4PathUpdates[ip4Path] = update - } - ip4Paths, ok := self.updateIp4Paths[update] - if !ok { - ip4Paths = map[Ip4Path]bool{} - self.updateIp4Paths[update] = ip4Paths - } - ip4Paths[ip4Path] = true - return update - case 6: - ip6Path := ipPath.ToIp6Path() - update, ok := self.ip6PathUpdates[ip6Path] - if !ok { - update = &multiClientChannelUpdate{} - self.ip6PathUpdates[ip6Path] = update - } - ip6Paths, ok := self.updateIp6Paths[update] - if !ok { - ip6Paths = map[Ip6Path]bool{} - self.updateIp6Paths[update] = ip6Paths - } - ip6Paths[ip6Path] = true - return update - default: - panic(fmt.Errorf("Bad protocol version %d", ipPath.Version)) - } - } - - updatePaths := func(previousClient *multiClientChannel, update *multiClientChannelUpdate) { - self.stateLock.Lock() - defer self.stateLock.Unlock() - - - client := update.client - - if previousClient != client { - if previousClient != nil { - delete(self.clientUpdates, previousClient) - } - if client != nil { - self.clientUpdates[client] = update - } - } - - if client == nil { - switch ipPath.Version { - case 4: - ip4Path := ipPath.ToIp4Path() - delete(self.ip4PathUpdates, ip4Path) - if ip4Paths, ok := self.updateIp4Paths[update]; ok { - delete(ip4Paths, ip4Path) - if len(ip4Paths) == 0 { - delete(self.updateIp4Paths, update) - } - } - case 6: - ip6Path := ipPath.ToIp6Path() - delete(self.ip6PathUpdates, ip6Path) - if ip6Paths, ok := self.updateIp6Paths[update]; ok { - delete(ip6Paths, ip6Path) - if len(ip6Paths) == 0 { - delete(self.updateIp6Paths, update) - } - } - default: - panic(fmt.Errorf("Bad protocol version %d", ipPath.Version)) - } - } - } - - // the client state lock can be acquired from inside the update lock - // ** important ** the update lock cannot be acquired from inside the client state lock - for { - // spin to acquire the correct update lock - update := reserveUpdate() - success := func()(bool) { - update.lock.Lock() - defer update.lock.Unlock() - - // update might have changed - if updateInLock := reserveUpdate(); update != updateInLock { - return false - } - - previousClient := update.client - callback(update) - updatePaths(previousClient, update) - return true - }() - if success { - return - } - } + reserveUpdate := func() *multiClientChannelUpdate { + self.stateLock.Lock() + defer self.stateLock.Unlock() + + switch ipPath.Version { + case 4: + ip4Path := ipPath.ToIp4Path() + update, ok := self.ip4PathUpdates[ip4Path] + if !ok { + update = &multiClientChannelUpdate{} + self.ip4PathUpdates[ip4Path] = update + } + ip4Paths, ok := self.updateIp4Paths[update] + if !ok { + ip4Paths = map[Ip4Path]bool{} + self.updateIp4Paths[update] = ip4Paths + } + ip4Paths[ip4Path] = true + return update + case 6: + ip6Path := ipPath.ToIp6Path() + update, ok := self.ip6PathUpdates[ip6Path] + if !ok { + update = &multiClientChannelUpdate{} + self.ip6PathUpdates[ip6Path] = update + } + ip6Paths, ok := self.updateIp6Paths[update] + if !ok { + ip6Paths = map[Ip6Path]bool{} + self.updateIp6Paths[update] = ip6Paths + } + ip6Paths[ip6Path] = true + return update + default: + panic(fmt.Errorf("Bad protocol version %d", ipPath.Version)) + } + } + + updatePaths := func(previousClient *multiClientChannel, update *multiClientChannelUpdate) { + self.stateLock.Lock() + defer self.stateLock.Unlock() + + client := update.client + + if previousClient != client { + if previousClient != nil { + delete(self.clientUpdates, previousClient) + } + if client != nil { + self.clientUpdates[client] = update + } + } + + if client == nil { + switch ipPath.Version { + case 4: + ip4Path := ipPath.ToIp4Path() + delete(self.ip4PathUpdates, ip4Path) + if ip4Paths, ok := self.updateIp4Paths[update]; ok { + delete(ip4Paths, ip4Path) + if len(ip4Paths) == 0 { + delete(self.updateIp4Paths, update) + } + } + case 6: + ip6Path := ipPath.ToIp6Path() + delete(self.ip6PathUpdates, ip6Path) + if ip6Paths, ok := self.updateIp6Paths[update]; ok { + delete(ip6Paths, ip6Path) + if len(ip6Paths) == 0 { + delete(self.updateIp6Paths, update) + } + } + default: + panic(fmt.Errorf("Bad protocol version %d", ipPath.Version)) + } + } + } + + // the client state lock can be acquired from inside the update lock + // ** important ** the update lock cannot be acquired from inside the client state lock + for { + // spin to acquire the correct update lock + update := reserveUpdate() + success := func() bool { + update.lock.Lock() + defer update.lock.Unlock() + + // update might have changed + if updateInLock := reserveUpdate(); update != updateInLock { + return false + } + + previousClient := update.client + callback(update) + updatePaths(previousClient, update) + return true + }() + if success { + return + } + } } // remove a client from all paths // this acts as a drop. it does not lock the client update func (self *RemoteUserNatMultiClient) removeClient(client *multiClientChannel) { - self.stateLock.Lock() - defer self.stateLock.Unlock() - - if update, ok := self.clientUpdates[client]; ok { - delete(self.clientUpdates, client) - - if ip4Paths, ok := self.updateIp4Paths[update]; ok { - delete(self.updateIp4Paths, update) - for ip4Path, _ := range ip4Paths { - delete(self.ip4PathUpdates, ip4Path) - } - } - - if ip6Paths, ok := self.updateIp6Paths[update]; ok { - delete(self.updateIp6Paths, update) - for ip6Path, _ := range ip6Paths { - delete(self.ip6PathUpdates, ip6Path) - } - } - } + self.stateLock.Lock() + defer self.stateLock.Unlock() + + if update, ok := self.clientUpdates[client]; ok { + delete(self.clientUpdates, client) + + if ip4Paths, ok := self.updateIp4Paths[update]; ok { + delete(self.updateIp4Paths, update) + for ip4Path, _ := range ip4Paths { + delete(self.ip4PathUpdates, ip4Path) + } + } + + if ip6Paths, ok := self.updateIp6Paths[update]; ok { + delete(self.updateIp6Paths, update) + for ip6Path, _ := range ip6Paths { + delete(self.ip6PathUpdates, ip6Path) + } + } + } } // `SendPacketFunction` func (self *RemoteUserNatMultiClient) SendPacket( - source TransferPath, - provideMode protocol.ProvideMode, - packet []byte, - timeout time.Duration, + source TransferPath, + provideMode protocol.ProvideMode, + packet []byte, + timeout time.Duration, ) (success bool) { - parsedPacket, err := newParsedPacket(packet) - if err != nil { - // bad packet - glog.Infof("[multi]send bad packet = %s\n", err) - success = false - return - } - - self.updateClientPath(parsedPacket.ipPath, func(update *multiClientChannelUpdate) { - enterTime := time.Now() - - if update.client != nil { - var err error - success, err = update.client.SendDetailed(parsedPacket, timeout) - if err == nil { - return - } - glog.Infof("[multi]send error = %s\n", err) - // find a new client - update.client = nil - } - - for { - orderedClients, removedClients := self.window.OrderedClients() - - for _, client := range removedClients { - glog.V(2).Infof("[multi]remove client %s->%s.\n", client.args.ClientId, client.args.Destination) - self.removeClient(client) - } - - for _, client := range orderedClients { - if client.Send(parsedPacket, 0) { - // lock the path to the client - update.client = client - success = true - return - } - } - - var retryTimeout time.Duration - if 0 <= timeout { - remainingTimeout := enterTime.Add(timeout).Sub(time.Now()) - - if remainingTimeout <= 0 { - // drop - success = false - return - } - - - retryTimeout = min(remainingTimeout, self.settings.WriteRetryTimeout) - - } else { - retryTimeout = self.settings.WriteRetryTimeout - } - - if 0 < len(orderedClients) { - // distribute the timeout evenly via wait - retryTimeoutPerClient := retryTimeout / time.Duration(len(orderedClients)) - for _, client := range orderedClients { - success, err = client.SendDetailed(parsedPacket, retryTimeoutPerClient) - if success && err == nil { - // lock the path to the client - update.client = client - success = true - return - } else if err != nil { - glog.Infof("[multi]send error = %s\n", err) - } - select { - case <- self.ctx.Done(): - // drop - success = false - return - default: - } - } - } else { - select { - case <- self.ctx.Done(): - // drop - success = false - return - case <- time.After(retryTimeout): - } - } - } - }) - return + parsedPacket, err := newParsedPacket(packet) + if err != nil { + // bad packet + glog.Infof("[multi]send bad packet = %s\n", err) + success = false + return + } + + self.updateClientPath(parsedPacket.ipPath, func(update *multiClientChannelUpdate) { + enterTime := time.Now() + + if update.client != nil { + var err error + success, err = update.client.SendDetailed(parsedPacket, timeout) + if err == nil { + return + } + glog.Infof("[multi]send error = %s\n", err) + // find a new client + update.client = nil + } + + for { + orderedClients, removedClients := self.window.OrderedClients() + + for _, client := range removedClients { + glog.V(2).Infof("[multi]remove client %s->%s.\n", client.args.ClientId, client.args.Destination) + self.removeClient(client) + } + + for _, client := range orderedClients { + if client.Send(parsedPacket, 0) { + // lock the path to the client + update.client = client + success = true + return + } + } + + var retryTimeout time.Duration + if 0 <= timeout { + remainingTimeout := enterTime.Add(timeout).Sub(time.Now()) + + if remainingTimeout <= 0 { + // drop + success = false + return + } + + retryTimeout = min(remainingTimeout, self.settings.WriteRetryTimeout) + + } else { + retryTimeout = self.settings.WriteRetryTimeout + } + + if 0 < len(orderedClients) { + // distribute the timeout evenly via wait + retryTimeoutPerClient := retryTimeout / time.Duration(len(orderedClients)) + for _, client := range orderedClients { + success, err = client.SendDetailed(parsedPacket, retryTimeoutPerClient) + if success && err == nil { + // lock the path to the client + update.client = client + success = true + return + } else if err != nil { + glog.Infof("[multi]send error = %s\n", err) + } + select { + case <-self.ctx.Done(): + // drop + success = false + return + default: + } + } + } else { + select { + case <-self.ctx.Done(): + // drop + success = false + return + case <-time.After(retryTimeout): + } + } + } + }) + return } func (self *RemoteUserNatMultiClient) Shuffle() { - self.window.shuffle() + self.window.shuffle() } func (self *RemoteUserNatMultiClient) Close() { - self.cancel() + self.cancel() } - type multiClientChannelUpdate struct { - lock sync.Mutex - client *multiClientChannel + lock sync.Mutex + client *multiClientChannel } - type parsedPacket struct { - packet []byte - ipPath *IpPath + packet []byte + ipPath *IpPath } func newParsedPacket(packet []byte) (*parsedPacket, error) { - ipPath, err := ParseIpPath(packet) - if err != nil { - return nil, err - } - return &parsedPacket{ - packet: packet, - ipPath: ipPath, - }, nil + ipPath, err := ParseIpPath(packet) + if err != nil { + return nil, err + } + return &parsedPacket{ + packet: packet, + ipPath: ipPath, + }, nil } - type MultiClientGeneratorClientArgs struct { - ClientId Id - ClientAuth *ClientAuth - P2pOnly bool + ClientId Id + ClientAuth *ClientAuth + P2pOnly bool } - func DefaultApiMultiClientGeneratorSettings() *ApiMultiClientGeneratorSettings { - return &ApiMultiClientGeneratorSettings{ - InitTimeout: 5 * time.Second, - } + return &ApiMultiClientGeneratorSettings{ + InitTimeout: 5 * time.Second, + } } - type ApiMultiClientGeneratorSettings struct { - InitTimeout time.Duration + InitTimeout time.Duration } - type ApiMultiClientGenerator struct { - specs []*ProviderSpec - excludeClientIds []Id - apiUrl string - byJwt string - platformUrl string - deviceDescription string - deviceSpec string - appVersion string - clientSettingsGenerator func()(*ClientSettings) - settings *ApiMultiClientGeneratorSettings + specs []*ProviderSpec + excludeClientIds []Id + apiUrl string + byJwt string + platformUrl string + deviceDescription string + deviceSpec string + appVersion string + clientSettingsGenerator func() *ClientSettings + settings *ApiMultiClientGeneratorSettings - api *BringYourApi + api *BringYourApi } func NewApiMultiClientGeneratorWithDefaults( - specs []*ProviderSpec, - excludeClientIds []Id, - apiUrl string, - byJwt string, - platformUrl string, - deviceDescription string, - deviceSpec string, - appVersion string, + specs []*ProviderSpec, + excludeClientIds []Id, + apiUrl string, + byJwt string, + platformUrl string, + deviceDescription string, + deviceSpec string, + appVersion string, ) *ApiMultiClientGenerator { - return NewApiMultiClientGenerator( - specs, - excludeClientIds, - apiUrl, - byJwt, - platformUrl, - deviceDescription, - deviceSpec, - appVersion, - DefaultClientSettings, - DefaultApiMultiClientGeneratorSettings(), - ) + return NewApiMultiClientGenerator( + specs, + excludeClientIds, + apiUrl, + byJwt, + platformUrl, + deviceDescription, + deviceSpec, + appVersion, + DefaultClientSettings, + DefaultApiMultiClientGeneratorSettings(), + ) } func NewApiMultiClientGenerator( - specs []*ProviderSpec, - excludeClientIds []Id, - apiUrl string, - byJwt string, - platformUrl string, - deviceDescription string, - deviceSpec string, - appVersion string, - clientSettingsGenerator func()(*ClientSettings), - settings *ApiMultiClientGeneratorSettings, + specs []*ProviderSpec, + excludeClientIds []Id, + apiUrl string, + byJwt string, + platformUrl string, + deviceDescription string, + deviceSpec string, + appVersion string, + clientSettingsGenerator func() *ClientSettings, + settings *ApiMultiClientGeneratorSettings, ) *ApiMultiClientGenerator { - api := NewBringYourApi(apiUrl) - api.SetByJwt(byJwt) - - return &ApiMultiClientGenerator{ - specs: specs, - excludeClientIds: excludeClientIds, - apiUrl: apiUrl, - byJwt: byJwt, - platformUrl: platformUrl, - deviceDescription: deviceDescription, - deviceSpec: deviceSpec, - appVersion: appVersion, - clientSettingsGenerator: clientSettingsGenerator, - settings: settings, - api: api, - } + api := NewBringYourApi(apiUrl) + api.SetByJwt(byJwt) + + return &ApiMultiClientGenerator{ + specs: specs, + excludeClientIds: excludeClientIds, + apiUrl: apiUrl, + byJwt: byJwt, + platformUrl: platformUrl, + deviceDescription: deviceDescription, + deviceSpec: deviceSpec, + appVersion: appVersion, + clientSettingsGenerator: clientSettingsGenerator, + settings: settings, + api: api, + } } func (self *ApiMultiClientGenerator) NextDestinations(count int, excludeDestinations []MultiHopId) (map[MultiHopId]ByteCount, error) { - excludeClientIds := slices.Clone(self.excludeClientIds) - excludeDestinationsIds := [][]Id{} - for _, excludeDestination := range excludeDestinations { - excludeDestinationsIds = append(excludeDestinationsIds, excludeDestination.Ids()) - } - findProviders2 := &FindProviders2Args{ - Specs: self.specs, - ExcludeClientIds: excludeClientIds, - ExcludeDestinations: excludeDestinationsIds, - Count: count, - } - - result, err := self.api.FindProviders2Sync(findProviders2) - if err != nil { - return nil, err - } - - clientIdEstimatedBytesPerSecond := map[MultiHopId]ByteCount{} - for _, provider := range result.Providers { - ids := []Id{} - if 0 < len(provider.IntermediaryIds) { - ids = append(ids, provider.IntermediaryIds...) - } - ids = append(ids, provider.ClientId) - // use the tail if the length exceeds the allowed maximum - if MaxMultihopLength < len(ids) { - ids = ids[len(ids)-MaxMultihopLength:len(ids)] - } - if destination, err := NewMultiHopId(ids...); err == nil { - clientIdEstimatedBytesPerSecond[destination] = provider.EstimatedBytesPerSecond - } - } - - return clientIdEstimatedBytesPerSecond, nil + excludeClientIds := slices.Clone(self.excludeClientIds) + excludeDestinationsIds := [][]Id{} + for _, excludeDestination := range excludeDestinations { + excludeDestinationsIds = append(excludeDestinationsIds, excludeDestination.Ids()) + } + findProviders2 := &FindProviders2Args{ + Specs: self.specs, + ExcludeClientIds: excludeClientIds, + ExcludeDestinations: excludeDestinationsIds, + Count: count, + } + + result, err := self.api.FindProviders2Sync(findProviders2) + if err != nil { + return nil, err + } + + clientIdEstimatedBytesPerSecond := map[MultiHopId]ByteCount{} + for _, provider := range result.Providers { + ids := []Id{} + if 0 < len(provider.IntermediaryIds) { + ids = append(ids, provider.IntermediaryIds...) + } + ids = append(ids, provider.ClientId) + // use the tail if the length exceeds the allowed maximum + if MaxMultihopLength < len(ids) { + ids = ids[len(ids)-MaxMultihopLength : len(ids)] + } + if destination, err := NewMultiHopId(ids...); err == nil { + clientIdEstimatedBytesPerSecond[destination] = provider.EstimatedBytesPerSecond + } + } + + return clientIdEstimatedBytesPerSecond, nil } func (self *ApiMultiClientGenerator) NewClientArgs() (*MultiClientGeneratorClientArgs, error) { - auth := func() (string, error) { - // note the derived client id will be inferred by the api jwt - authNetworkClient := &AuthNetworkClientArgs{ - Description: self.deviceDescription, - DeviceSpec: self.deviceSpec, - } - - result, err := self.api.AuthNetworkClientSync(authNetworkClient) - if err != nil { - return "", err - } - - if result.Error != nil { - return "", errors.New(result.Error.Message) - } - - return result.ByClientJwt, nil - } - - if byJwtStr, err := auth(); err == nil { - byJwt, err := ParseByJwtUnverified(byJwtStr) - if err != nil { - // in this case we cannot clean up the client because we don't know the client id - panic(err) - } - - clientAuth := &ClientAuth{ - ByJwt: byJwtStr, - InstanceId: NewId(), - AppVersion: self.appVersion, - } - return &MultiClientGeneratorClientArgs{ - ClientId: byJwt.ClientId, - ClientAuth: clientAuth, - }, nil - } else { - return nil, err - } + auth := func() (string, error) { + // note the derived client id will be inferred by the api jwt + authNetworkClient := &AuthNetworkClientArgs{ + Description: self.deviceDescription, + DeviceSpec: self.deviceSpec, + } + + result, err := self.api.AuthNetworkClientSync(authNetworkClient) + if err != nil { + return "", err + } + + if result.Error != nil { + return "", errors.New(result.Error.Message) + } + + return result.ByClientJwt, nil + } + + if byJwtStr, err := auth(); err == nil { + byJwt, err := ParseByJwtUnverified(byJwtStr) + if err != nil { + // in this case we cannot clean up the client because we don't know the client id + panic(err) + } + + clientAuth := &ClientAuth{ + ByJwt: byJwtStr, + InstanceId: NewId(), + AppVersion: self.appVersion, + } + return &MultiClientGeneratorClientArgs{ + ClientId: byJwt.ClientId, + ClientAuth: clientAuth, + }, nil + } else { + return nil, err + } } func (self *ApiMultiClientGenerator) RemoveClientArgs(args *MultiClientGeneratorClientArgs) { - removeNetworkClient := &RemoveNetworkClientArgs{ - ClientId: args.ClientId, - } + removeNetworkClient := &RemoveNetworkClientArgs{ + ClientId: args.ClientId, + } - self.api.RemoveNetworkClient(removeNetworkClient, NewApiCallback(func(result *RemoveNetworkClientResult, err error) { - })) + self.api.RemoveNetworkClient(removeNetworkClient, NewApiCallback(func(result *RemoveNetworkClientResult, err error) { + })) } func (self *ApiMultiClientGenerator) RemoveClientWithArgs(client *Client, args *MultiClientGeneratorClientArgs) { - self.RemoveClientArgs(args) + self.RemoveClientArgs(args) } func (self *ApiMultiClientGenerator) NewClientSettings() *ClientSettings { - return self.clientSettingsGenerator() + return self.clientSettingsGenerator() } func (self *ApiMultiClientGenerator) NewClient( - ctx context.Context, - args *MultiClientGeneratorClientArgs, - clientSettings *ClientSettings, + ctx context.Context, + args *MultiClientGeneratorClientArgs, + clientSettings *ClientSettings, ) (*Client, error) { - byJwt, err := ParseByJwtUnverified(args.ClientAuth.ByJwt) - if err != nil { - return nil, err - } - clientOob := NewApiOutOfBandControl(ctx, args.ClientAuth.ByJwt, self.apiUrl) - client := NewClient(ctx, byJwt.ClientId, clientOob, clientSettings) - settings := DefaultPlatformTransportSettings() - if args.P2pOnly { - settings.TransportGenerator = func()(sendTransport Transport, receiveTransport Transport) { - // only use the platform transport for control - sendTransport = NewSendClientTransport(DestinationId(ControlId)) - receiveTransport = NewReceiveGatewayTransport() - return - } - } - NewPlatformTransportWithDefaultDialer( - client.Ctx(), - self.platformUrl, - args.ClientAuth, - client.RouteManager(), - settings, - ) - // enable return traffic for this client - ack := make(chan struct{}) - client.ContractManager().SetProvideModesWithReturnTrafficWithAckCallback( - map[protocol.ProvideMode]bool{}, - func(err error) { - close(ack) - }, - ) - select { - case <- ack: - case <- time.After(self.settings.InitTimeout): - client.Cancel() - return nil, errors.New("Could not enable return traffic for client.") - } - return client, nil + byJwt, err := ParseByJwtUnverified(args.ClientAuth.ByJwt) + if err != nil { + return nil, err + } + clientOob := NewApiOutOfBandControl(ctx, args.ClientAuth.ByJwt, self.apiUrl) + client := NewClient(ctx, byJwt.ClientId, clientOob, clientSettings) + settings := DefaultPlatformTransportSettings() + if args.P2pOnly { + settings.TransportGenerator = func() (sendTransport Transport, receiveTransport Transport) { + // only use the platform transport for control + sendTransport = NewSendClientTransport(DestinationId(ControlId)) + receiveTransport = NewReceiveGatewayTransport() + return + } + } + NewPlatformTransportWithDefaultDialer( + client.Ctx(), + self.platformUrl, + args.ClientAuth, + client.RouteManager(), + settings, + ) + // enable return traffic for this client + ack := make(chan struct{}) + client.ContractManager().SetProvideModesWithReturnTrafficWithAckCallback( + map[protocol.ProvideMode]bool{}, + func(err error) { + close(ack) + }, + ) + select { + case <-ack: + case <-time.After(self.settings.InitTimeout): + client.Cancel() + return nil, errors.New("Could not enable return traffic for client.") + } + return client, nil } - type multiClientWindow struct { - ctx context.Context - cancel context.CancelFunc + ctx context.Context + cancel context.CancelFunc - generator MultiClientGenerator - receivePacketCallback ReceivePacketFunction + generator MultiClientGenerator + receivePacketCallback ReceivePacketFunction - settings *MultiClientSettings + settings *MultiClientSettings - clientChannelArgs chan *multiClientChannelArgs + clientChannelArgs chan *multiClientChannelArgs - monitor *RemoteUserNatMultiClientMonitor + monitor *RemoteUserNatMultiClientMonitor - stateLock sync.Mutex - destinationClients map[MultiHopId]*multiClientChannel + stateLock sync.Mutex + destinationClients map[MultiHopId]*multiClientChannel } func newMultiClientWindow( - ctx context.Context, - cancel context.CancelFunc, - generator MultiClientGenerator, - receivePacketCallback ReceivePacketFunction, - settings *MultiClientSettings, + ctx context.Context, + cancel context.CancelFunc, + generator MultiClientGenerator, + receivePacketCallback ReceivePacketFunction, + settings *MultiClientSettings, ) *multiClientWindow { - window := &multiClientWindow{ - ctx: ctx, - cancel: cancel, - generator: generator, - receivePacketCallback: receivePacketCallback, - settings: settings, - clientChannelArgs: make(chan *multiClientChannelArgs, settings.WindowSizeMin), - monitor: NewRemoteUserNatMultiClientMonitor(&settings.RemoteUserNatMultiClientMonitorSettings), - destinationClients: map[MultiHopId]*multiClientChannel{}, - } + window := &multiClientWindow{ + ctx: ctx, + cancel: cancel, + generator: generator, + receivePacketCallback: receivePacketCallback, + settings: settings, + clientChannelArgs: make(chan *multiClientChannelArgs, settings.WindowSizeMin), + monitor: NewRemoteUserNatMultiClientMonitor(&settings.RemoteUserNatMultiClientMonitorSettings), + destinationClients: map[MultiHopId]*multiClientChannel{}, + } - go HandleError(window.randomEnumerateClientArgs, cancel) - go HandleError(window.resize, cancel) + go HandleError(window.randomEnumerateClientArgs, cancel) + go HandleError(window.resize, cancel) - return window + return window } func (self *multiClientWindow) randomEnumerateClientArgs() { - defer func() { - close(self.clientChannelArgs) - - // drain the channel - func() { - for { - select { - case args, ok := <- self.clientChannelArgs: - if !ok { - return - } - self.generator.RemoveClientArgs(&args.MultiClientGeneratorClientArgs) - } - } - }() - }() - - // continually reset the visited set when there are no more - visitedDestinations := map[MultiHopId]bool{} - for { - destinationEstimatedBytesPerSecond := map[MultiHopId]ByteCount{} - for { - next := func(count int) (map[MultiHopId]ByteCount, error) { - return self.generator.NextDestinations( - count, - maps.Keys(visitedDestinations), - ) - } - - nextDestinationEstimatedBytesPerSecond, err := next(1) - if err != nil { - select { - case <- self.ctx.Done(): - return - case <- time.After(self.settings.WindowEnumerateErrorTimeout): - glog.V(2).Infof("[multi]window enumerate error timeout.\n") - } - } else if 0 < len(nextDestinationEstimatedBytesPerSecond) { - for destination, estimatedBytesPerSecond := range nextDestinationEstimatedBytesPerSecond { - destinationEstimatedBytesPerSecond[destination] = estimatedBytesPerSecond - visitedDestinations[destination] = true - } - break - } else { - // reset - visitedDestinations = map[MultiHopId]bool{} - select { - case <- self.ctx.Done(): - return - case <- time.After(self.settings.WindowEnumerateEmptyTimeout): - glog.V(2).Infof("[multi]window enumerate empty timeout.\n") - } - } - } - - // remove destinations that are already in the window - func() { - self.stateLock.Lock() - defer self.stateLock.Unlock() - for destination, _ := range self.destinationClients { - delete(destinationEstimatedBytesPerSecond, destination) - } - }() - - for destination, estimatedBytesPerSecond := range destinationEstimatedBytesPerSecond { - if clientArgs, err := self.generator.NewClientArgs(); err == nil { - args := &multiClientChannelArgs{ - Destination: destination, - EstimatedBytesPerSecond: estimatedBytesPerSecond, - MultiClientGeneratorClientArgs: *clientArgs, - } - select { - case <- self.ctx.Done(): - self.generator.RemoveClientArgs(clientArgs) - return - case self.clientChannelArgs <- args: - } - } else { - glog.Infof("[multi]create client args error = %s\n", err) - } - - } - } + defer func() { + close(self.clientChannelArgs) + + // drain the channel + func() { + for { + select { + case args, ok := <-self.clientChannelArgs: + if !ok { + return + } + self.generator.RemoveClientArgs(&args.MultiClientGeneratorClientArgs) + } + } + }() + }() + + // continually reset the visited set when there are no more + visitedDestinations := map[MultiHopId]bool{} + for { + destinationEstimatedBytesPerSecond := map[MultiHopId]ByteCount{} + for { + next := func(count int) (map[MultiHopId]ByteCount, error) { + return self.generator.NextDestinations( + count, + maps.Keys(visitedDestinations), + ) + } + + nextDestinationEstimatedBytesPerSecond, err := next(1) + if err != nil { + select { + case <-self.ctx.Done(): + return + case <-time.After(self.settings.WindowEnumerateErrorTimeout): + glog.V(2).Infof("[multi]window enumerate error timeout.\n") + } + } else if 0 < len(nextDestinationEstimatedBytesPerSecond) { + for destination, estimatedBytesPerSecond := range nextDestinationEstimatedBytesPerSecond { + destinationEstimatedBytesPerSecond[destination] = estimatedBytesPerSecond + visitedDestinations[destination] = true + } + break + } else { + // reset + visitedDestinations = map[MultiHopId]bool{} + select { + case <-self.ctx.Done(): + return + case <-time.After(self.settings.WindowEnumerateEmptyTimeout): + glog.V(2).Infof("[multi]window enumerate empty timeout.\n") + } + } + } + + // remove destinations that are already in the window + func() { + self.stateLock.Lock() + defer self.stateLock.Unlock() + for destination, _ := range self.destinationClients { + delete(destinationEstimatedBytesPerSecond, destination) + } + }() + + for destination, estimatedBytesPerSecond := range destinationEstimatedBytesPerSecond { + if clientArgs, err := self.generator.NewClientArgs(); err == nil { + args := &multiClientChannelArgs{ + Destination: destination, + EstimatedBytesPerSecond: estimatedBytesPerSecond, + MultiClientGeneratorClientArgs: *clientArgs, + } + select { + case <-self.ctx.Done(): + self.generator.RemoveClientArgs(clientArgs) + return + case self.clientChannelArgs <- args: + } + } else { + glog.Infof("[multi]create client args error = %s\n", err) + } + + } + } } func (self *multiClientWindow) resize() { - // based on the most recent expand failure - expandOvershotScale := float64(1.0) - for { - startTime := time.Now() - - clients := []*multiClientChannel{} - - maxSourceCount := 0 - weights := map[*multiClientChannel]float32{} - durations := map[*multiClientChannel]time.Duration{} - - for _, client := range self.clients() { - if stats, err := client.WindowStats(); err == nil { - clients = append(clients, client) - maxSourceCount = max(maxSourceCount, stats.sourceCount) - // byte count per second - weights[client] = float32(stats.ByteCountPerSecond()) - durations[client] = stats.duration - } - // else ignore the client for optimization - // it will get cleaned up on the next call to `OrderedClients` - } - - slices.SortFunc(clients, func(a *multiClientChannel, b *multiClientChannel)(int) { - // descending weight - aWeight := weights[a] - bWeight := weights[b] - if aWeight < bWeight { - return 1 - } else if bWeight < aWeight { - return -1 - } else { - return 0 - } - }) - - targetWindowSize := min( - self.settings.WindowSizeMax, - max( - self.settings.WindowSizeMin, - int(math.Ceil(float64(maxSourceCount) * self.settings.WindowSizeReconnectScale)), - ), - ) - - // expand and collapse have scale thresholds to avoid jittery resizing - // too much resing wastes device resources - expandWindowSize := min( - self.settings.WindowSizeMax, - max( - self.settings.WindowSizeMin, - int(math.Ceil(self.settings.WindowExpandScale * float64(len(clients)))), - ), - ) - collapseWindowSize := int(math.Ceil(self.settings.WindowCollapseScale * float64(len(clients)))) - - collapseLowestWeighted := func(windowSize int)(int) { - // try to remove the lowest weighted clients to resize the window to `windowSize` - // clients in the graceperiod or with activity cannot be removed - - self.stateLock.Lock() - defer self.stateLock.Unlock() - - n := 0 - - collapseClients := clients[windowSize:] - clients = clients[:windowSize] - for _, client := range collapseClients { - if self.settings.StatsWindowGraceperiod <= durations[client] && weights[client] <= 0 { - client.Cancel() - n += 1 - } else { - clients = append(clients, client) - } - } - - destinationClients := map[MultiHopId]*multiClientChannel{} - for _, client := range clients { - destinationClients[client.Destination()] = client - } - self.destinationClients = destinationClients - - return n - } - - p2pOnlyWindowSize := 0 - for _, client := range clients { - if client.IsP2pOnly() { - p2pOnlyWindowSize += 1 - } - } - if expandWindowSize <= targetWindowSize && len(clients) < expandWindowSize || p2pOnlyWindowSize < self.settings.WindowSizeMinP2pOnly { - // collapse badly performing clients before expanding - n := collapseLowestWeighted(0) - if 0 < n { - glog.Infof("[multi]window optimize -%d ->%d\n", n, len(clients)) - } - - // expand - n = expandWindowSize - len(clients) - self.monitor.AddWindowExpandEvent(len(clients), expandWindowSize) - overN := int(math.Ceil(expandOvershotScale * float64(n))) - glog.Infof("[multi]window expand +%d(%d) %d->%d\n", n, overN, len(clients), expandWindowSize) - self.expand(len(clients), p2pOnlyWindowSize, expandWindowSize, overN) - - // evaluate the next overshot scale - func () { - self.stateLock.Lock() - defer self.stateLock.Unlock() - n = len(self.destinationClients) - len(clients) - }() - if n <= 0 { - expandOvershotScale = self.settings.WindowExpandMaxOvershotScale - } else { - // overN = s * n - expandOvershotScale = min( - self.settings.WindowExpandMaxOvershotScale, - float64(overN) / float64(n), - ) - } - } else if targetWindowSize <= collapseWindowSize && collapseWindowSize < len(clients) { - self.monitor.AddWindowExpandEvent(len(clients), collapseWindowSize) - n := collapseLowestWeighted(collapseWindowSize) - if 0 < n { - glog.Infof("[multi]window collapse -%d ->%d\n", n, len(clients)) - } - self.monitor.AddWindowExpandEvent(len(clients), collapseWindowSize) - } else { - self.monitor.AddWindowExpandEvent(len(clients), len(clients)) - glog.Infof("[multi]window stable =%d\n", len(clients)) - } - - timeout := self.settings.WindowResizeTimeout - time.Now().Sub(startTime) - if timeout <= 0 { - select { - case <- self.ctx.Done(): - return - default: - } - } else { - select { - case <- self.ctx.Done(): - return - case <- time.After(timeout): - } - } - } + // based on the most recent expand failure + expandOvershotScale := float64(1.0) + for { + startTime := time.Now() + + clients := []*multiClientChannel{} + + maxSourceCount := 0 + weights := map[*multiClientChannel]float32{} + durations := map[*multiClientChannel]time.Duration{} + + for _, client := range self.clients() { + if stats, err := client.WindowStats(); err == nil { + clients = append(clients, client) + maxSourceCount = max(maxSourceCount, stats.sourceCount) + // byte count per second + weights[client] = float32(stats.ByteCountPerSecond()) + durations[client] = stats.duration + } + // else ignore the client for optimization + // it will get cleaned up on the next call to `OrderedClients` + } + + slices.SortFunc(clients, func(a *multiClientChannel, b *multiClientChannel) int { + // descending weight + aWeight := weights[a] + bWeight := weights[b] + if aWeight < bWeight { + return 1 + } else if bWeight < aWeight { + return -1 + } else { + return 0 + } + }) + + targetWindowSize := min( + self.settings.WindowSizeMax, + max( + self.settings.WindowSizeMin, + int(math.Ceil(float64(maxSourceCount)*self.settings.WindowSizeReconnectScale)), + ), + ) + + // expand and collapse have scale thresholds to avoid jittery resizing + // too much resing wastes device resources + expandWindowSize := min( + self.settings.WindowSizeMax, + max( + self.settings.WindowSizeMin, + int(math.Ceil(self.settings.WindowExpandScale*float64(len(clients)))), + ), + ) + collapseWindowSize := int(math.Ceil(self.settings.WindowCollapseScale * float64(len(clients)))) + + collapseLowestWeighted := func(windowSize int) int { + // try to remove the lowest weighted clients to resize the window to `windowSize` + // clients in the graceperiod or with activity cannot be removed + + self.stateLock.Lock() + defer self.stateLock.Unlock() + + n := 0 + + collapseClients := clients[windowSize:] + clients = clients[:windowSize] + for _, client := range collapseClients { + if self.settings.StatsWindowGraceperiod <= durations[client] && weights[client] <= 0 { + client.Cancel() + n += 1 + } else { + clients = append(clients, client) + } + } + + destinationClients := map[MultiHopId]*multiClientChannel{} + for _, client := range clients { + destinationClients[client.Destination()] = client + } + self.destinationClients = destinationClients + + return n + } + + p2pOnlyWindowSize := 0 + for _, client := range clients { + if client.IsP2pOnly() { + p2pOnlyWindowSize += 1 + } + } + if expandWindowSize <= targetWindowSize && len(clients) < expandWindowSize || p2pOnlyWindowSize < self.settings.WindowSizeMinP2pOnly { + // collapse badly performing clients before expanding + n := collapseLowestWeighted(0) + if 0 < n { + glog.Infof("[multi]window optimize -%d ->%d\n", n, len(clients)) + } + + // expand + n = expandWindowSize - len(clients) + self.monitor.AddWindowExpandEvent(len(clients), expandWindowSize) + overN := int(math.Ceil(expandOvershotScale * float64(n))) + glog.Infof("[multi]window expand +%d(%d) %d->%d\n", n, overN, len(clients), expandWindowSize) + self.expand(len(clients), p2pOnlyWindowSize, expandWindowSize, overN) + + // evaluate the next overshot scale + func() { + self.stateLock.Lock() + defer self.stateLock.Unlock() + n = len(self.destinationClients) - len(clients) + }() + if n <= 0 { + expandOvershotScale = self.settings.WindowExpandMaxOvershotScale + } else { + // overN = s * n + expandOvershotScale = min( + self.settings.WindowExpandMaxOvershotScale, + float64(overN)/float64(n), + ) + } + } else if targetWindowSize <= collapseWindowSize && collapseWindowSize < len(clients) { + self.monitor.AddWindowExpandEvent(len(clients), collapseWindowSize) + n := collapseLowestWeighted(collapseWindowSize) + if 0 < n { + glog.Infof("[multi]window collapse -%d ->%d\n", n, len(clients)) + } + self.monitor.AddWindowExpandEvent(len(clients), collapseWindowSize) + } else { + self.monitor.AddWindowExpandEvent(len(clients), len(clients)) + glog.Infof("[multi]window stable =%d\n", len(clients)) + } + + timeout := self.settings.WindowResizeTimeout - time.Now().Sub(startTime) + if timeout <= 0 { + select { + case <-self.ctx.Done(): + return + default: + } + } else { + select { + case <-self.ctx.Done(): + return + case <-time.After(timeout): + } + } + } } func (self *multiClientWindow) expand(currentWindowSize int, currentP2pOnlyWindowSize int, targetWindowSize int, n int) { - mutex := sync.Mutex{} - addedCount := 0 - - endTime := time.Now().Add(self.settings.WindowExpandTimeout) - pendingPingDones := []chan struct{}{} - added := 0 - addedP2pOnly := 0 - for i := 0; i < n; i += 1 { - timeout := endTime.Sub(time.Now()) - if timeout < 0 { - glog.Infof("[multi]expand window timeout\n") - return - } - - select { - case <- self.ctx.Done(): - return - // case <- update: - // // continue - case args, ok := <- self.clientChannelArgs: - if !ok { - return - } - func() { - self.stateLock.Lock() - defer self.stateLock.Unlock() - _, ok = self.destinationClients[args.Destination] - }() - - if ok { - // already have a client in the window for this destination - self.generator.RemoveClientArgs(&args.MultiClientGeneratorClientArgs) - } else { - // randomly set to p2p only to meet the minimum requirement - if !args.MultiClientGeneratorClientArgs.P2pOnly { - a := max(self.settings.WindowSizeMin - (currentWindowSize + added), 0) - b := max(self.settings.WindowSizeMinP2pOnly - (currentP2pOnlyWindowSize + addedP2pOnly), 0) - var p2pOnlyP float32 - if a + b == 0 { - p2pOnlyP = 0 - } else { - p2pOnlyP = float32(b) / float32(a + b) - } - args.MultiClientGeneratorClientArgs.P2pOnly = mathrand.Float32() < p2pOnlyP - } - - client, err := newMultiClientChannel( - self.ctx, - args, - self.generator, - self.receivePacketCallback, - self.settings, - ) - if err == nil { - added += 1 - if client.IsP2pOnly() { - addedP2pOnly += 1 - } - - self.monitor.AddProviderEvent(args.ClientId, ProviderStateInEvaluation) - - // send an initial ping on the client and let the ack timeout close it - pingDone := make(chan struct{}) - success, err := client.SendDetailedMessage( - &protocol.IpPing{}, - self.settings.PingWriteTimeout, - func (err error) { - defer close(pingDone) - if err == nil { - glog.Infof("[multi]expand new client\n") - self.monitor.AddProviderEvent(args.ClientId, ProviderStateAdded) - func () { - self.stateLock.Lock() - defer self.stateLock.Unlock() - self.destinationClients[args.Destination] = client - }() - func() { - mutex.Lock() - defer mutex.Unlock() - addedCount += 1 - self.monitor.AddWindowExpandEvent(currentWindowSize + addedCount, targetWindowSize) - }() - } else { - glog.Infof("[multi]create ping error = %s\n", err) - client.Cancel() - self.monitor.AddProviderEvent(args.ClientId, ProviderStateEvaluationFailed) - } - }, - ) - if err != nil { - glog.Infof("[multi]create client ping error = %s\n", err) - client.Cancel() - } else if !success { - client.Cancel() - self.monitor.AddProviderEvent(args.ClientId, ProviderStateEvaluationFailed) - } else { - // async wait for the ping - pendingPingDones = append(pendingPingDones, pingDone) - go func() { - select { - case <- pingDone: - case <- time.After(self.settings.PingTimeout): - glog.V(2).Infof("[multi]expand window timeout waiting for ping\n") - client.Cancel() - } - }() - } - } else { - glog.Infof("[multi]create client error = %s\n", err) - self.generator.RemoveClientArgs(&args.MultiClientGeneratorClientArgs) - self.monitor.AddProviderEvent(args.ClientId, ProviderStateEvaluationFailed) - } - } - case <- time.After(timeout): - glog.V(2).Infof("[multi]expand window timeout waiting for args\n") - } - } - - // wait for pending pings - for _, pingDone := range pendingPingDones { - select { - case <- self.ctx.Done(): - return - case <- pingDone: - } - } - - return + mutex := sync.Mutex{} + addedCount := 0 + + endTime := time.Now().Add(self.settings.WindowExpandTimeout) + pendingPingDones := []chan struct{}{} + added := 0 + addedP2pOnly := 0 + for i := 0; i < n; i += 1 { + timeout := endTime.Sub(time.Now()) + if timeout < 0 { + glog.Infof("[multi]expand window timeout\n") + return + } + + select { + case <-self.ctx.Done(): + return + // case <- update: + // // continue + case args, ok := <-self.clientChannelArgs: + if !ok { + return + } + func() { + self.stateLock.Lock() + defer self.stateLock.Unlock() + _, ok = self.destinationClients[args.Destination] + }() + + if ok { + // already have a client in the window for this destination + self.generator.RemoveClientArgs(&args.MultiClientGeneratorClientArgs) + } else { + // randomly set to p2p only to meet the minimum requirement + if !args.MultiClientGeneratorClientArgs.P2pOnly { + a := max(self.settings.WindowSizeMin-(currentWindowSize+added), 0) + b := max(self.settings.WindowSizeMinP2pOnly-(currentP2pOnlyWindowSize+addedP2pOnly), 0) + var p2pOnlyP float32 + if a+b == 0 { + p2pOnlyP = 0 + } else { + p2pOnlyP = float32(b) / float32(a+b) + } + args.MultiClientGeneratorClientArgs.P2pOnly = mathrand.Float32() < p2pOnlyP + } + + client, err := newMultiClientChannel( + self.ctx, + args, + self.generator, + self.receivePacketCallback, + self.settings, + ) + if err == nil { + added += 1 + if client.IsP2pOnly() { + addedP2pOnly += 1 + } + + self.monitor.AddProviderEvent(args.ClientId, ProviderStateInEvaluation) + + // send an initial ping on the client and let the ack timeout close it + pingDone := make(chan struct{}) + success, err := client.SendDetailedMessage( + &protocol.IpPing{}, + self.settings.PingWriteTimeout, + func(err error) { + defer close(pingDone) + if err == nil { + glog.Infof("[multi]expand new client\n") + self.monitor.AddProviderEvent(args.ClientId, ProviderStateAdded) + func() { + self.stateLock.Lock() + defer self.stateLock.Unlock() + self.destinationClients[args.Destination] = client + }() + func() { + mutex.Lock() + defer mutex.Unlock() + addedCount += 1 + self.monitor.AddWindowExpandEvent(currentWindowSize+addedCount, targetWindowSize) + }() + } else { + glog.Infof("[multi]create ping error = %s\n", err) + client.Cancel() + self.monitor.AddProviderEvent(args.ClientId, ProviderStateEvaluationFailed) + } + }, + ) + if err != nil { + glog.Infof("[multi]create client ping error = %s\n", err) + client.Cancel() + } else if !success { + client.Cancel() + self.monitor.AddProviderEvent(args.ClientId, ProviderStateEvaluationFailed) + } else { + // async wait for the ping + pendingPingDones = append(pendingPingDones, pingDone) + go func() { + select { + case <-pingDone: + case <-time.After(self.settings.PingTimeout): + glog.V(2).Infof("[multi]expand window timeout waiting for ping\n") + client.Cancel() + } + }() + } + } else { + glog.Infof("[multi]create client error = %s\n", err) + self.generator.RemoveClientArgs(&args.MultiClientGeneratorClientArgs) + self.monitor.AddProviderEvent(args.ClientId, ProviderStateEvaluationFailed) + } + } + case <-time.After(timeout): + glog.V(2).Infof("[multi]expand window timeout waiting for args\n") + } + } + + // wait for pending pings + for _, pingDone := range pendingPingDones { + select { + case <-self.ctx.Done(): + return + case <-pingDone: + } + } + + return } func (self *multiClientWindow) shuffle() { - for _, client := range self.clients() { - client.Cancel() - } + for _, client := range self.clients() { + client.Cancel() + } } func (self *multiClientWindow) clients() []*multiClientChannel { - self.stateLock.Lock() - defer self.stateLock.Unlock() - return maps.Values(self.destinationClients) + self.stateLock.Lock() + defer self.stateLock.Unlock() + return maps.Values(self.destinationClients) } func (self *multiClientWindow) OrderedClients() ([]*multiClientChannel, []*multiClientChannel) { - clients := []*multiClientChannel{} - - removedClients := []*multiClientChannel{} - weights := map[*multiClientChannel]float32{} - durations := map[*multiClientChannel]time.Duration{} - - for _, client := range self.clients() { - if stats, err := client.WindowStats(); err != nil { - glog.Infof("[multi]remove client = %s\n", err) - removedClients = append(removedClients, client) - } else { - clients = append(clients, client) - weights[client] = float32(stats.ByteCountPerSecond()) - durations[client] = stats.duration - } - } - - if 0 < len(removedClients) { - func() { - self.stateLock.Lock() - defer self.stateLock.Unlock() - for _, client := range removedClients { - client.Cancel() - delete(self.destinationClients, client.Destination()) - } - }() - for _, client := range removedClients { - self.monitor.AddProviderEvent(client.ClientId(), ProviderStateRemoved) - } - } - - // iterate and adjust weights for clients with weights >= 0 - nonNegativeClients := []*multiClientChannel{} - for _, client := range clients { - if weight := weights[client]; 0 <= weight { - if duration := durations[client]; duration < self.settings.StatsWindowGraceperiod { - // use the estimate - weights[client] = float32(client.EstimatedByteCountPerSecond()) - } else if 0 == weight { - // not used, use the estimate - weights[client] = float32(client.EstimatedByteCountPerSecond()) - } - nonNegativeClients = append(nonNegativeClients, client) - } - } - - if glog.V(1) { - self.statsSampleWeights(weights) - } - - WeightedShuffleWithEntropy(nonNegativeClients, weights, self.settings.StatsWindowEntropy) - - return nonNegativeClients, removedClients + clients := []*multiClientChannel{} + + removedClients := []*multiClientChannel{} + weights := map[*multiClientChannel]float32{} + durations := map[*multiClientChannel]time.Duration{} + + for _, client := range self.clients() { + if stats, err := client.WindowStats(); err != nil { + glog.Infof("[multi]remove client = %s\n", err) + removedClients = append(removedClients, client) + } else { + clients = append(clients, client) + weights[client] = float32(stats.ByteCountPerSecond()) + durations[client] = stats.duration + } + } + + if 0 < len(removedClients) { + func() { + self.stateLock.Lock() + defer self.stateLock.Unlock() + for _, client := range removedClients { + client.Cancel() + delete(self.destinationClients, client.Destination()) + } + }() + for _, client := range removedClients { + self.monitor.AddProviderEvent(client.ClientId(), ProviderStateRemoved) + } + } + + // iterate and adjust weights for clients with weights >= 0 + nonNegativeClients := []*multiClientChannel{} + for _, client := range clients { + if weight := weights[client]; 0 <= weight { + if duration := durations[client]; duration < self.settings.StatsWindowGraceperiod { + // use the estimate + weights[client] = float32(client.EstimatedByteCountPerSecond()) + } else if 0 == weight { + // not used, use the estimate + weights[client] = float32(client.EstimatedByteCountPerSecond()) + } + nonNegativeClients = append(nonNegativeClients, client) + } + } + + if glog.V(1) { + self.statsSampleWeights(weights) + } + + WeightedShuffleWithEntropy(nonNegativeClients, weights, self.settings.StatsWindowEntropy) + + return nonNegativeClients, removedClients } func (self *multiClientWindow) statsSampleWeights(weights map[*multiClientChannel]float32) { - // randonly sample log statistics for weights - if mathrand.Intn(self.settings.StatsSampleWeightsCount) == 0 { - // sample the weights - weightValues := maps.Values(weights) - slices.SortFunc(weightValues, func(a float32, b float32)(int) { - // descending - if a < b { - return 1 - } else if b < a { - return -1 - } else { - return 0 - } - }) - net := float32(0) - for _, weight := range weightValues { - net += weight - } - if 0 < net { - var sb strings.Builder - netThresh := float32(0.99) - netp := float32(0) - netCount := 0 - for i, weight := range weightValues { - p := 100 * weight / net - netp += p - netCount += 1 - if 0 < i { - sb.WriteString(" ") - } - sb.WriteString(fmt.Sprintf("[%d]%.2f", i, p)) - if netThresh * 100 <= netp { - break - } - } - - glog.Infof("[multi]sample weights: %s (+%d more in window <%.0f%%)\n", sb.String(), len(weights) - netCount, 100 * (1 - netThresh)) - } else { - glog.Infof("[multi]sample weights: zero (%d in window)\n", len(weights)) - } - } + // randonly sample log statistics for weights + if mathrand.Intn(self.settings.StatsSampleWeightsCount) == 0 { + // sample the weights + weightValues := maps.Values(weights) + slices.SortFunc(weightValues, func(a float32, b float32) int { + // descending + if a < b { + return 1 + } else if b < a { + return -1 + } else { + return 0 + } + }) + net := float32(0) + for _, weight := range weightValues { + net += weight + } + if 0 < net { + var sb strings.Builder + netThresh := float32(0.99) + netp := float32(0) + netCount := 0 + for i, weight := range weightValues { + p := 100 * weight / net + netp += p + netCount += 1 + if 0 < i { + sb.WriteString(" ") + } + sb.WriteString(fmt.Sprintf("[%d]%.2f", i, p)) + if netThresh*100 <= netp { + break + } + } + + glog.Infof("[multi]sample weights: %s (+%d more in window <%.0f%%)\n", sb.String(), len(weights)-netCount, 100*(1-netThresh)) + } else { + glog.Infof("[multi]sample weights: zero (%d in window)\n", len(weights)) + } + } } - type multiClientChannelArgs struct { - MultiClientGeneratorClientArgs + MultiClientGeneratorClientArgs - Destination MultiHopId - EstimatedBytesPerSecond ByteCount + Destination MultiHopId + EstimatedBytesPerSecond ByteCount } - type multiClientEventType int + const ( - multiClientEventTypeAck multiClientEventType = 1 - multiClientEventTypeNack multiClientEventType = 2 - multiClientEventTypeError multiClientEventType = 3 - multiClientEventTypeSource multiClientEventType = 4 + multiClientEventTypeAck multiClientEventType = 1 + multiClientEventTypeNack multiClientEventType = 2 + multiClientEventTypeError multiClientEventType = 3 + multiClientEventTypeSource multiClientEventType = 4 ) - type multiClientEventBucket struct { - createTime time.Time - eventTime time.Time + createTime time.Time + eventTime time.Time - sendAckCount int - sendAckByteCount ByteCount - sendNackCount int - sendNackByteCount ByteCount - receiveAckCount int - receiveAckByteCount ByteCount - sendAckTime time.Time - errs []error - ip4Paths map[Ip4Path]bool - ip6Paths map[Ip6Path]bool + sendAckCount int + sendAckByteCount ByteCount + sendNackCount int + sendNackByteCount ByteCount + receiveAckCount int + receiveAckByteCount ByteCount + sendAckTime time.Time + errs []error + ip4Paths map[Ip4Path]bool + ip6Paths map[Ip6Path]bool } func newMultiClientEventBucket() *multiClientEventBucket { - now := time.Now() - return &multiClientEventBucket{ - createTime: now, - eventTime: now, - } + now := time.Now() + return &multiClientEventBucket{ + createTime: now, + eventTime: now, + } } type clientWindowStats struct { - sourceCount int - sendAckCount int - sendAckByteCount ByteCount - sendNackCount int - sendNackByteCount ByteCount - receiveAckCount int - receiveAckByteCount ByteCount - ackByteCount ByteCount - duration time.Duration - sendAckDuration time.Duration + sourceCount int + sendAckCount int + sendAckByteCount ByteCount + sendNackCount int + sendNackByteCount ByteCount + receiveAckCount int + receiveAckByteCount ByteCount + ackByteCount ByteCount + duration time.Duration + sendAckDuration time.Duration - // internal - bucketCount int + // internal + bucketCount int } func (self *clientWindowStats) ByteCountPerSecond() ByteCount { - seconds := float64(self.duration / time.Second) - if seconds <= 0 { - return ByteCount(0) - } - return ByteCount(float64(self.sendAckByteCount - self.sendNackByteCount + self.receiveAckByteCount) / seconds) + seconds := float64(self.duration / time.Second) + if seconds <= 0 { + return ByteCount(0) + } + return ByteCount(float64(self.sendAckByteCount-self.sendNackByteCount+self.receiveAckByteCount) / seconds) } - type multiClientChannel struct { - ctx context.Context - cancel context.CancelFunc + ctx context.Context + cancel context.CancelFunc - args *multiClientChannelArgs + args *multiClientChannelArgs - api *BringYourApi + api *BringYourApi - receivePacketCallback ReceivePacketFunction + receivePacketCallback ReceivePacketFunction - settings *MultiClientSettings + settings *MultiClientSettings - // sourceFilter map[TransferPath]bool + // sourceFilter map[TransferPath]bool - client *Client + client *Client - stateLock sync.Mutex - eventBuckets []*multiClientEventBucket - // destination -> source -> count - ip4DestinationSourceCount map[Ip4Path]map[Ip4Path]int - ip6DestinationSourceCount map[Ip6Path]map[Ip6Path]int - packetStats *clientWindowStats - endErr error + stateLock sync.Mutex + eventBuckets []*multiClientEventBucket + // destination -> source -> count + ip4DestinationSourceCount map[Ip4Path]map[Ip4Path]int + ip6DestinationSourceCount map[Ip6Path]map[Ip6Path]int + packetStats *clientWindowStats + endErr error - clientReceiveUnsub func() + clientReceiveUnsub func() } func newMultiClientChannel( - ctx context.Context, - args *multiClientChannelArgs, - generator MultiClientGenerator, - receivePacketCallback ReceivePacketFunction, - settings *MultiClientSettings, + ctx context.Context, + args *multiClientChannelArgs, + generator MultiClientGenerator, + receivePacketCallback ReceivePacketFunction, + settings *MultiClientSettings, ) (*multiClientChannel, error) { - cancelCtx, cancel := context.WithCancel(ctx) - - clientSettings := generator.NewClientSettings() - clientSettings.SendBufferSettings.AckTimeout = settings.AckTimeout - - client, err := generator.NewClient( - cancelCtx, - &args.MultiClientGeneratorClientArgs, - clientSettings, - ) - if err != nil { - return nil, err - } - go HandleError(func() { - select { - case <- cancelCtx.Done(): - case <- client.Done(): - } - client.Cancel() - generator.RemoveClientWithArgs(client, &args.MultiClientGeneratorClientArgs) - }, cancel) - - // sourceFilter := map[TransferPath]bool{ - // Path{ClientId:args.DestinationId}: true, - // } - - clientChannel := &multiClientChannel{ - ctx: cancelCtx, - cancel: cancel, - args: args, - receivePacketCallback: receivePacketCallback, - settings: settings, - // sourceFilter: sourceFilter, - client: client, - eventBuckets: []*multiClientEventBucket{}, - ip4DestinationSourceCount: map[Ip4Path]map[Ip4Path]int{}, - ip6DestinationSourceCount: map[Ip6Path]map[Ip6Path]int{}, - packetStats: &clientWindowStats{}, - endErr: nil, - } - go HandleError(clientChannel.detectBlackhole, cancel) - - clientReceiveUnsub := client.AddReceiveCallback(clientChannel.clientReceive) - clientChannel.clientReceiveUnsub = clientReceiveUnsub - - return clientChannel, nil + cancelCtx, cancel := context.WithCancel(ctx) + + clientSettings := generator.NewClientSettings() + clientSettings.SendBufferSettings.AckTimeout = settings.AckTimeout + + client, err := generator.NewClient( + cancelCtx, + &args.MultiClientGeneratorClientArgs, + clientSettings, + ) + if err != nil { + return nil, err + } + go HandleError(func() { + select { + case <-cancelCtx.Done(): + case <-client.Done(): + } + client.Cancel() + generator.RemoveClientWithArgs(client, &args.MultiClientGeneratorClientArgs) + }, cancel) + + // sourceFilter := map[TransferPath]bool{ + // Path{ClientId:args.DestinationId}: true, + // } + + clientChannel := &multiClientChannel{ + ctx: cancelCtx, + cancel: cancel, + args: args, + receivePacketCallback: receivePacketCallback, + settings: settings, + // sourceFilter: sourceFilter, + client: client, + eventBuckets: []*multiClientEventBucket{}, + ip4DestinationSourceCount: map[Ip4Path]map[Ip4Path]int{}, + ip6DestinationSourceCount: map[Ip6Path]map[Ip6Path]int{}, + packetStats: &clientWindowStats{}, + endErr: nil, + } + go HandleError(clientChannel.detectBlackhole, cancel) + + clientReceiveUnsub := client.AddReceiveCallback(clientChannel.clientReceive) + clientChannel.clientReceiveUnsub = clientReceiveUnsub + + return clientChannel, nil } func (self *multiClientChannel) ClientId() Id { - return self.client.ClientId() + return self.client.ClientId() } func (self *multiClientChannel) IsP2pOnly() bool { - return self.args.MultiClientGeneratorClientArgs.P2pOnly + return self.args.MultiClientGeneratorClientArgs.P2pOnly } func (self *multiClientChannel) Send(parsedPacket *parsedPacket, timeout time.Duration) bool { - success, err := self.SendDetailed(parsedPacket, timeout) - return success && err == nil + success, err := self.SendDetailed(parsedPacket, timeout) + return success && err == nil } func (self *multiClientChannel) SendDetailed(parsedPacket *parsedPacket, timeout time.Duration) (bool, error) { - ipPacketToProvider := &protocol.IpPacketToProvider{ - IpPacket: &protocol.IpPacket{ - PacketBytes: parsedPacket.packet, - }, - } - if frame, err := ToFrame(ipPacketToProvider); err != nil { - self.addError(err) - return false, err - } else { - packetByteCount := ByteCount(len(parsedPacket.packet)) - self.addSendNack(packetByteCount) - self.addSource(parsedPacket.ipPath) - ackCallback := func(err error) { - if err == nil { - self.addSendAck(packetByteCount) - } else { - self.addError(err) - } - } - - opts := []any{ - ForceStream(), - } - switch parsedPacket.ipPath.Protocol { - case IpProtocolUdp: - opts = append(opts, NoAck()) - } - return self.client.SendMultiHopWithTimeoutDetailed( - frame, - self.args.Destination, - ackCallback, - timeout, - opts..., - ) - } + ipPacketToProvider := &protocol.IpPacketToProvider{ + IpPacket: &protocol.IpPacket{ + PacketBytes: parsedPacket.packet, + }, + } + if frame, err := ToFrame(ipPacketToProvider); err != nil { + self.addError(err) + return false, err + } else { + packetByteCount := ByteCount(len(parsedPacket.packet)) + self.addSendNack(packetByteCount) + self.addSource(parsedPacket.ipPath) + ackCallback := func(err error) { + if err == nil { + self.addSendAck(packetByteCount) + } else { + self.addError(err) + } + } + + opts := []any{ + ForceStream(), + } + switch parsedPacket.ipPath.Protocol { + case IpProtocolUdp: + opts = append(opts, NoAck()) + } + return self.client.SendMultiHopWithTimeoutDetailed( + frame, + self.args.Destination, + ackCallback, + timeout, + opts..., + ) + } } func (self *multiClientChannel) SendDetailedMessage(message proto.Message, timeout time.Duration, ackCallback func(error)) (bool, error) { - if frame, err := ToFrame(message); err != nil { - return false, err - } else { - return self.client.SendMultiHopWithTimeoutDetailed( - frame, - self.args.Destination, - ackCallback, - timeout, - ForceStream(), - ) - } + if frame, err := ToFrame(message); err != nil { + return false, err + } else { + return self.client.SendMultiHopWithTimeoutDetailed( + frame, + self.args.Destination, + ackCallback, + timeout, + ForceStream(), + ) + } } func (self *multiClientChannel) EstimatedByteCountPerSecond() ByteCount { - return self.args.EstimatedBytesPerSecond + return self.args.EstimatedBytesPerSecond } func (self *multiClientChannel) Done() <-chan struct{} { - return self.ctx.Done() + return self.ctx.Done() } func (self *multiClientChannel) Destination() MultiHopId { - return self.args.Destination + return self.args.Destination } func (self *multiClientChannel) detectBlackhole() { - // within a timeout window, if there are sent data but none received, - // error out. This is similar to an ack timeout. - defer self.cancel() - - for { - if windowStats, err := self.WindowStats(); err != nil { - return - } else { - timeout := self.settings.BlackholeTimeout - windowStats.sendAckDuration - if timeout <= 0 { - timeout = self.settings.BlackholeTimeout - - if 0 < windowStats.sendAckCount && windowStats.receiveAckCount <= 0 { - // the client has sent data but received nothing back - // this looks like a blackhole - glog.Infof("[multi]routing %s blackhole: %d %dB <> %d %dB\n", - self.args.Destination, - windowStats.sendAckCount, - windowStats.sendAckByteCount, - windowStats.receiveAckCount, - windowStats.receiveAckByteCount, - ) - self.addError(fmt.Errorf("Blackhole (%d %dB)", - windowStats.sendAckCount, - windowStats.sendAckByteCount, - )) - return - } else { - glog.Infof( - "[multi]routing ok %s: %d %dB <> %d %dB\n", - self.args.Destination, - windowStats.sendAckCount, - windowStats.sendAckByteCount, - windowStats.receiveAckCount, - windowStats.receiveAckByteCount, - ) - } - } - - select { - case <- self.ctx.Done(): - return - case <- time.After(timeout): - } - } - } + // within a timeout window, if there are sent data but none received, + // error out. This is similar to an ack timeout. + defer self.cancel() + + for { + if windowStats, err := self.WindowStats(); err != nil { + return + } else { + timeout := self.settings.BlackholeTimeout - windowStats.sendAckDuration + if timeout <= 0 { + timeout = self.settings.BlackholeTimeout + + if 0 < windowStats.sendAckCount && windowStats.receiveAckCount <= 0 { + // the client has sent data but received nothing back + // this looks like a blackhole + glog.Infof("[multi]routing %s blackhole: %d %dB <> %d %dB\n", + self.args.Destination, + windowStats.sendAckCount, + windowStats.sendAckByteCount, + windowStats.receiveAckCount, + windowStats.receiveAckByteCount, + ) + self.addError(fmt.Errorf("Blackhole (%d %dB)", + windowStats.sendAckCount, + windowStats.sendAckByteCount, + )) + return + } else { + glog.Infof( + "[multi]routing ok %s: %d %dB <> %d %dB\n", + self.args.Destination, + windowStats.sendAckCount, + windowStats.sendAckByteCount, + windowStats.receiveAckCount, + windowStats.receiveAckByteCount, + ) + } + } + + select { + case <-self.ctx.Done(): + return + case <-time.After(timeout): + } + } + } } func (self *multiClientChannel) eventBucket() *multiClientEventBucket { - // must be called with stateLock + // must be called with stateLock - now := time.Now() + now := time.Now() - var eventBucket *multiClientEventBucket - if n := len(self.eventBuckets); 0 < n { - eventBucket = self.eventBuckets[n - 1] - if eventBucket.createTime.Add(self.settings.StatsWindowBucketDuration).Before(now) { - // expired - eventBucket = nil - } - } - - if eventBucket == nil { - eventBucket = newMultiClientEventBucket() - self.eventBuckets = append(self.eventBuckets, eventBucket) - } + var eventBucket *multiClientEventBucket + if n := len(self.eventBuckets); 0 < n { + eventBucket = self.eventBuckets[n-1] + if eventBucket.createTime.Add(self.settings.StatsWindowBucketDuration).Before(now) { + // expired + eventBucket = nil + } + } - eventBucket.eventTime = now + if eventBucket == nil { + eventBucket = newMultiClientEventBucket() + self.eventBuckets = append(self.eventBuckets, eventBucket) + } - return eventBucket + eventBucket.eventTime = now + + return eventBucket } func (self *multiClientChannel) addSendNack(ackByteCount ByteCount) { - self.stateLock.Lock() - defer self.stateLock.Unlock() + self.stateLock.Lock() + defer self.stateLock.Unlock() - self.packetStats.sendNackCount += 1 - self.packetStats.sendNackByteCount += ackByteCount + self.packetStats.sendNackCount += 1 + self.packetStats.sendNackByteCount += ackByteCount - eventBucket := self.eventBucket() - eventBucket.sendNackCount += 1 - eventBucket.sendNackByteCount += ackByteCount + eventBucket := self.eventBucket() + eventBucket.sendNackCount += 1 + eventBucket.sendNackByteCount += ackByteCount - self.coalesceEventBuckets() + self.coalesceEventBuckets() } func (self *multiClientChannel) addSendAck(ackByteCount ByteCount) { - self.stateLock.Lock() - defer self.stateLock.Unlock() + self.stateLock.Lock() + defer self.stateLock.Unlock() - self.packetStats.sendNackCount -= 1 - self.packetStats.sendNackByteCount -= ackByteCount - self.packetStats.sendAckCount += 1 - self.packetStats.sendAckByteCount += ackByteCount + self.packetStats.sendNackCount -= 1 + self.packetStats.sendNackByteCount -= ackByteCount + self.packetStats.sendAckCount += 1 + self.packetStats.sendAckByteCount += ackByteCount - eventBucket := self.eventBucket() - if eventBucket.sendAckCount == 0 { - eventBucket.sendAckTime = time.Now() - } - eventBucket.sendAckCount += 1 - eventBucket.sendAckByteCount += ackByteCount + eventBucket := self.eventBucket() + if eventBucket.sendAckCount == 0 { + eventBucket.sendAckTime = time.Now() + } + eventBucket.sendAckCount += 1 + eventBucket.sendAckByteCount += ackByteCount - self.coalesceEventBuckets() + self.coalesceEventBuckets() } func (self *multiClientChannel) addReceiveAck(ackByteCount ByteCount) { - self.stateLock.Lock() - defer self.stateLock.Unlock() + self.stateLock.Lock() + defer self.stateLock.Unlock() - self.packetStats.receiveAckCount += 1 - self.packetStats.receiveAckByteCount += ackByteCount + self.packetStats.receiveAckCount += 1 + self.packetStats.receiveAckByteCount += ackByteCount - eventBucket := self.eventBucket() - eventBucket.receiveAckCount += 1 - eventBucket.receiveAckByteCount += ackByteCount + eventBucket := self.eventBucket() + eventBucket.receiveAckCount += 1 + eventBucket.receiveAckByteCount += ackByteCount - self.coalesceEventBuckets() + self.coalesceEventBuckets() } func (self *multiClientChannel) addSource(ipPath *IpPath) { - self.stateLock.Lock() - defer self.stateLock.Unlock() - - source := ipPath.Source() - destination := ipPath.Destination() - switch source.Version { - case 4: - sourceCount, ok := self.ip4DestinationSourceCount[destination.ToIp4Path()] - if !ok { - sourceCount = map[Ip4Path]int{} - self.ip4DestinationSourceCount[destination.ToIp4Path()] = sourceCount - } - sourceCount[source.ToIp4Path()] += 1 - case 6: - sourceCount, ok := self.ip6DestinationSourceCount[destination.ToIp6Path()] - if !ok { - sourceCount = map[Ip6Path]int{} - self.ip6DestinationSourceCount[destination.ToIp6Path()] = sourceCount - } - sourceCount[source.ToIp6Path()] += 1 - default: - panic(fmt.Errorf("Bad protocol version %d", source.Version)) - } - - eventBucket := self.eventBucket() - switch ipPath.Version { - case 4: - if eventBucket.ip4Paths == nil { - eventBucket.ip4Paths = map[Ip4Path]bool{} - } - eventBucket.ip4Paths[ipPath.ToIp4Path()] = true - case 6: - if eventBucket.ip6Paths == nil { - eventBucket.ip6Paths = map[Ip6Path]bool{} - } - eventBucket.ip6Paths[ipPath.ToIp6Path()] = true - default: - panic(fmt.Errorf("Bad protocol version %d", source.Version)) - } - - self.coalesceEventBuckets() + self.stateLock.Lock() + defer self.stateLock.Unlock() + + source := ipPath.Source() + destination := ipPath.Destination() + switch source.Version { + case 4: + sourceCount, ok := self.ip4DestinationSourceCount[destination.ToIp4Path()] + if !ok { + sourceCount = map[Ip4Path]int{} + self.ip4DestinationSourceCount[destination.ToIp4Path()] = sourceCount + } + sourceCount[source.ToIp4Path()] += 1 + case 6: + sourceCount, ok := self.ip6DestinationSourceCount[destination.ToIp6Path()] + if !ok { + sourceCount = map[Ip6Path]int{} + self.ip6DestinationSourceCount[destination.ToIp6Path()] = sourceCount + } + sourceCount[source.ToIp6Path()] += 1 + default: + panic(fmt.Errorf("Bad protocol version %d", source.Version)) + } + + eventBucket := self.eventBucket() + switch ipPath.Version { + case 4: + if eventBucket.ip4Paths == nil { + eventBucket.ip4Paths = map[Ip4Path]bool{} + } + eventBucket.ip4Paths[ipPath.ToIp4Path()] = true + case 6: + if eventBucket.ip6Paths == nil { + eventBucket.ip6Paths = map[Ip6Path]bool{} + } + eventBucket.ip6Paths[ipPath.ToIp6Path()] = true + default: + panic(fmt.Errorf("Bad protocol version %d", source.Version)) + } + + self.coalesceEventBuckets() } func (self *multiClientChannel) addError(err error) { - self.stateLock.Lock() - defer self.stateLock.Unlock() + self.stateLock.Lock() + defer self.stateLock.Unlock() - if self.endErr == nil { - self.endErr = err - } + if self.endErr == nil { + self.endErr = err + } - eventBucket := self.eventBucket() - eventBucket.errs = append(eventBucket.errs, err) + eventBucket := self.eventBucket() + eventBucket.errs = append(eventBucket.errs, err) - self.coalesceEventBuckets() + self.coalesceEventBuckets() } func (self *multiClientChannel) coalesceEventBuckets() { - // must be called with `stateLock` - - // if there is no activity (no new buckets), keep historical buckets around - minBucketCount := 1 + int(self.settings.StatsWindowDuration / self.settings.StatsWindowBucketDuration) - - windowStart := time.Now().Add(-self.settings.StatsWindowDuration) - - // remove events before the window start - i := 0 - for i < len(self.eventBuckets) && minBucketCount < len(self.eventBuckets) { - eventBucket := self.eventBuckets[i] - if windowStart.Before(eventBucket.eventTime) { - break - } - - self.packetStats.sendAckCount -= eventBucket.sendAckCount - self.packetStats.sendAckByteCount -= eventBucket.sendAckByteCount - self.packetStats.receiveAckCount -= eventBucket.receiveAckCount - self.packetStats.receiveAckByteCount -= eventBucket.receiveAckByteCount - - for ip4Path, _ := range eventBucket.ip4Paths { - source := ip4Path.Source() - destination := ip4Path.Destination() - - sourceCount, ok := self.ip4DestinationSourceCount[destination] - if ok { - count := sourceCount[source] - if count - 1 <= 0 { - delete(sourceCount, source) - } else { - sourceCount[source] = count - 1 - } - if len(sourceCount) == 0 { - delete(self.ip4DestinationSourceCount, destination) - } - } - } - - for ip6Path, _ := range eventBucket.ip6Paths { - source := ip6Path.Source() - destination := ip6Path.Destination() - - sourceCount, ok := self.ip6DestinationSourceCount[destination] - if ok { - count := sourceCount[source] - if count - 1 <= 0 { - delete(sourceCount, source) - } else { - sourceCount[source] = count - 1 - } - if len(sourceCount) == 0 { - delete(self.ip6DestinationSourceCount, destination) - } - } - } - - self.eventBuckets[i] = nil - i += 1 - } - if 0 < i { - self.eventBuckets = self.eventBuckets[i:] - } + // must be called with `stateLock` + + // if there is no activity (no new buckets), keep historical buckets around + minBucketCount := 1 + int(self.settings.StatsWindowDuration/self.settings.StatsWindowBucketDuration) + + windowStart := time.Now().Add(-self.settings.StatsWindowDuration) + + // remove events before the window start + i := 0 + for i < len(self.eventBuckets) && minBucketCount < len(self.eventBuckets) { + eventBucket := self.eventBuckets[i] + if windowStart.Before(eventBucket.eventTime) { + break + } + + self.packetStats.sendAckCount -= eventBucket.sendAckCount + self.packetStats.sendAckByteCount -= eventBucket.sendAckByteCount + self.packetStats.receiveAckCount -= eventBucket.receiveAckCount + self.packetStats.receiveAckByteCount -= eventBucket.receiveAckByteCount + + for ip4Path, _ := range eventBucket.ip4Paths { + source := ip4Path.Source() + destination := ip4Path.Destination() + + sourceCount, ok := self.ip4DestinationSourceCount[destination] + if ok { + count := sourceCount[source] + if count-1 <= 0 { + delete(sourceCount, source) + } else { + sourceCount[source] = count - 1 + } + if len(sourceCount) == 0 { + delete(self.ip4DestinationSourceCount, destination) + } + } + } + + for ip6Path, _ := range eventBucket.ip6Paths { + source := ip6Path.Source() + destination := ip6Path.Destination() + + sourceCount, ok := self.ip6DestinationSourceCount[destination] + if ok { + count := sourceCount[source] + if count-1 <= 0 { + delete(sourceCount, source) + } else { + sourceCount[source] = count - 1 + } + if len(sourceCount) == 0 { + delete(self.ip6DestinationSourceCount, destination) + } + } + } + + self.eventBuckets[i] = nil + i += 1 + } + if 0 < i { + self.eventBuckets = self.eventBuckets[i:] + } } func (self *multiClientChannel) WindowStats() (*clientWindowStats, error) { - return self.windowStatsWithCoalesce(true) + return self.windowStatsWithCoalesce(true) } func (self *multiClientChannel) windowStatsWithCoalesce(coalesce bool) (*clientWindowStats, error) { - self.stateLock.Lock() - defer self.stateLock.Unlock() - - if coalesce { - self.coalesceEventBuckets() - } - - duration := time.Duration(0) - if 0 < len(self.eventBuckets) { - duration = time.Now().Sub(self.eventBuckets[0].createTime) - } - sendAckDuration := time.Duration(0) - for _, eventBucket := range self.eventBuckets { - if 0 < eventBucket.sendAckCount { - sendAckDuration = time.Now().Sub(eventBucket.sendAckTime) - break - } - } - - - // public internet resource ports - isPublicPort := func(port int)(bool) { - switch port { - case 443: - return true - default: - return false - } - } - - - netSourceCounts := []int{} - for ip4Path, sourceCounts := range self.ip4DestinationSourceCount { - if isPublicPort(ip4Path.DestinationPort) { - netSourceCounts = append(netSourceCounts, len(sourceCounts)) - } - } - for ip6Path, sourceCounts := range self.ip6DestinationSourceCount { - if isPublicPort(ip6Path.DestinationPort) { - netSourceCounts = append(netSourceCounts, len(sourceCounts)) - } - } - slices.Sort(netSourceCounts) - maxSourceCount := 0 - if selectionIndex := int(math.Ceil(self.settings.StatsSourceCountSelection * float64(len(netSourceCounts) - 1))); selectionIndex < len(netSourceCounts) { - maxSourceCount = netSourceCounts[selectionIndex] - } - if glog.V(2) { - for ip4Path, sourceCounts := range self.ip4DestinationSourceCount { - if isPublicPort(ip4Path.DestinationPort) { - if len(sourceCounts) == maxSourceCount { - glog.Infof("[multi]max source count %d = %v\n", maxSourceCount, ip4Path) - } - } - } - for ip6Path, sourceCounts := range self.ip6DestinationSourceCount { - if isPublicPort(ip6Path.DestinationPort) { - if len(sourceCounts) == maxSourceCount { - glog.Infof("[multi]max source count %d = %v\n", maxSourceCount, ip6Path) - } - } - } - } - stats := &clientWindowStats{ - sourceCount: maxSourceCount, - sendAckCount: self.packetStats.sendAckCount, - sendNackCount: self.packetStats.sendNackCount, - sendAckByteCount: self.packetStats.sendAckByteCount, - sendNackByteCount: self.packetStats.sendNackByteCount, - receiveAckCount: self.packetStats.receiveAckCount, - receiveAckByteCount: self.packetStats.receiveAckByteCount, - duration: duration, - sendAckDuration: sendAckDuration, - bucketCount: len(self.eventBuckets), - } - err := self.endErr - - return stats, err + self.stateLock.Lock() + defer self.stateLock.Unlock() + + if coalesce { + self.coalesceEventBuckets() + } + + duration := time.Duration(0) + if 0 < len(self.eventBuckets) { + duration = time.Now().Sub(self.eventBuckets[0].createTime) + } + sendAckDuration := time.Duration(0) + for _, eventBucket := range self.eventBuckets { + if 0 < eventBucket.sendAckCount { + sendAckDuration = time.Now().Sub(eventBucket.sendAckTime) + break + } + } + + // public internet resource ports + isPublicPort := func(port int) bool { + switch port { + case 443: + return true + default: + return false + } + } + + netSourceCounts := []int{} + for ip4Path, sourceCounts := range self.ip4DestinationSourceCount { + if isPublicPort(ip4Path.DestinationPort) { + netSourceCounts = append(netSourceCounts, len(sourceCounts)) + } + } + for ip6Path, sourceCounts := range self.ip6DestinationSourceCount { + if isPublicPort(ip6Path.DestinationPort) { + netSourceCounts = append(netSourceCounts, len(sourceCounts)) + } + } + slices.Sort(netSourceCounts) + maxSourceCount := 0 + if selectionIndex := int(math.Ceil(self.settings.StatsSourceCountSelection * float64(len(netSourceCounts)-1))); selectionIndex < len(netSourceCounts) { + maxSourceCount = netSourceCounts[selectionIndex] + } + if glog.V(2) { + for ip4Path, sourceCounts := range self.ip4DestinationSourceCount { + if isPublicPort(ip4Path.DestinationPort) { + if len(sourceCounts) == maxSourceCount { + glog.Infof("[multi]max source count %d = %v\n", maxSourceCount, ip4Path) + } + } + } + for ip6Path, sourceCounts := range self.ip6DestinationSourceCount { + if isPublicPort(ip6Path.DestinationPort) { + if len(sourceCounts) == maxSourceCount { + glog.Infof("[multi]max source count %d = %v\n", maxSourceCount, ip6Path) + } + } + } + } + stats := &clientWindowStats{ + sourceCount: maxSourceCount, + sendAckCount: self.packetStats.sendAckCount, + sendNackCount: self.packetStats.sendNackCount, + sendAckByteCount: self.packetStats.sendAckByteCount, + sendNackByteCount: self.packetStats.sendNackByteCount, + receiveAckCount: self.packetStats.receiveAckCount, + receiveAckByteCount: self.packetStats.receiveAckByteCount, + duration: duration, + sendAckDuration: sendAckDuration, + bucketCount: len(self.eventBuckets), + } + err := self.endErr + + return stats, err } // `connect.ReceiveFunction` func (self *multiClientChannel) clientReceive(source TransferPath, frames []*protocol.Frame, provideMode protocol.ProvideMode) { - select { - case <- self.ctx.Done(): - return - default: - } - - // only process frames from the destinations - // if allow := self.sourceFilter[source]; !allow { - // glog.V(2).Infof("[multi]receive drop %d %s<-\n", len(frames), self.args.DestinationId) - // return - // } - - for _, frame := range frames { - switch frame.MessageType { - case protocol.MessageType_IpIpPacketFromProvider: - if ipPacketFromProvider_, err := FromFrame(frame); err == nil { - ipPacketFromProvider := ipPacketFromProvider_.(*protocol.IpPacketFromProvider) - - packet := ipPacketFromProvider.IpPacket.PacketBytes - - self.addReceiveAck(ByteCount(len(packet))) - - self.receivePacketCallback(source, IpProtocolUnknown, packet) - } else { - glog.V(2).Infof("[multi]receive drop %s<- = %s\n", self.args.Destination, err) - } - default: - // unknown message, drop - } - } + select { + case <-self.ctx.Done(): + return + default: + } + + // only process frames from the destinations + // if allow := self.sourceFilter[source]; !allow { + // glog.V(2).Infof("[multi]receive drop %d %s<-\n", len(frames), self.args.DestinationId) + // return + // } + + for _, frame := range frames { + switch frame.MessageType { + case protocol.MessageType_IpIpPacketFromProvider: + if ipPacketFromProvider_, err := FromFrame(frame); err == nil { + ipPacketFromProvider := ipPacketFromProvider_.(*protocol.IpPacketFromProvider) + + packet := ipPacketFromProvider.IpPacket.PacketBytes + + self.addReceiveAck(ByteCount(len(packet))) + + self.receivePacketCallback(source, IpProtocolUnknown, packet) + } else { + glog.V(2).Infof("[multi]receive drop %s<- = %s\n", self.args.Destination, err) + } + default: + // unknown message, drop + } + } } func (self *multiClientChannel) Cancel() { - self.addError(errors.New("Done.")) - self.cancel() - self.client.Cancel() + self.addError(errors.New("Done.")) + self.cancel() + self.client.Cancel() } func (self *multiClientChannel) Close() { - self.addError(errors.New("Done.")) - self.cancel() - self.client.Close() + self.addError(errors.New("Done.")) + self.cancel() + self.client.Close() - self.clientReceiveUnsub() + self.clientReceiveUnsub() } - - diff --git a/connect/ip_test.go b/connect/ip_test.go index 5ac8139..9a20179 100644 --- a/connect/ip_test.go +++ b/connect/ip_test.go @@ -2,45 +2,40 @@ package connect import ( "context" - "testing" - "time" "encoding/binary" "net" "reflect" + "testing" + "time" + // "sync" "fmt" "github.com/google/gopacket" - "github.com/google/gopacket/layers" + "github.com/google/gopacket/layers" - "github.com/go-playground/assert/v2" + "github.com/go-playground/assert/v2" - "bringyour.com/protocol" + "github.com/bringyour/connect/protocol" ) - func TestClientUdp4(t *testing.T) { testClient(t, testingNewClient, udp4Packet, (*IpPath).ToIp4Path) } - func TestClientTcp4(t *testing.T) { testClient(t, testingNewClient, tcp4Packet, (*IpPath).ToIp4Path) } - func TestClientUdp6(t *testing.T) { testClient(t, testingNewClient, udp6Packet, (*IpPath).ToIp6Path) } - func TestClientTcp6(t *testing.T) { testClient(t, testingNewClient, tcp6Packet, (*IpPath).ToIp6Path) } - -type PacketGeneratorFunction func(int, int, int, int)([]byte, []byte) - +type PacketGeneratorFunction func(int, int, int, int) ([]byte, []byte) func TestUdp4Path(t *testing.T) { packet, _ := udp4Packet(1, 1, 1, 1) @@ -50,25 +45,24 @@ func TestUdp4Path(t *testing.T) { assert.Equal(t, IpProtocolUdp, ipPath.Protocol) assert.Equal(t, &IpPath{ - Version: 4, - Protocol: IpProtocolUdp, - SourceIp: net.IPv4(byte(72), byte(0), byte(0), byte(1)).To4(), - SourcePort: 40000 + 1, - DestinationIp: net.IPv4(byte(72), byte(1 + 1), byte(1 + 1), byte(1 + 1)).To4(), - DestinationPort: 443, + Version: 4, + Protocol: IpProtocolUdp, + SourceIp: net.IPv4(byte(72), byte(0), byte(0), byte(1)).To4(), + SourcePort: 40000 + 1, + DestinationIp: net.IPv4(byte(72), byte(1+1), byte(1+1), byte(1+1)).To4(), + DestinationPort: 443, }, ipPath) ip4Path := ipPath.ToIp4Path() assert.Equal(t, Ip4Path{ - Protocol: IpProtocolUdp, - SourceIp: [4]byte(net.IPv4(byte(72), byte(0), byte(0), byte(1)).To4()), - SourcePort: 40000 + 1, - DestinationIp: [4]byte(net.IPv4(byte(72), byte(1 + 1), byte(1 + 1), byte(1 + 1)).To4()), - DestinationPort: 443, + Protocol: IpProtocolUdp, + SourceIp: [4]byte(net.IPv4(byte(72), byte(0), byte(0), byte(1)).To4()), + SourcePort: 40000 + 1, + DestinationIp: [4]byte(net.IPv4(byte(72), byte(1+1), byte(1+1), byte(1+1)).To4()), + DestinationPort: 443, }, ip4Path) } - func TestTcp4Path(t *testing.T) { packet, _ := tcp4Packet(1, 1, 1, 1) ipPath, err := ParseIpPath(packet) @@ -77,25 +71,24 @@ func TestTcp4Path(t *testing.T) { assert.Equal(t, IpProtocolTcp, ipPath.Protocol) assert.Equal(t, &IpPath{ - Version: 4, - Protocol: IpProtocolTcp, - SourceIp: net.IPv4(byte(72), byte(0), byte(0), byte(1)).To4(), - SourcePort: 40000 + 1, - DestinationIp: net.IPv4(byte(72), byte(1 + 1), byte(1 + 1), byte(1 + 1)).To4(), - DestinationPort: 443, + Version: 4, + Protocol: IpProtocolTcp, + SourceIp: net.IPv4(byte(72), byte(0), byte(0), byte(1)).To4(), + SourcePort: 40000 + 1, + DestinationIp: net.IPv4(byte(72), byte(1+1), byte(1+1), byte(1+1)).To4(), + DestinationPort: 443, }, ipPath) ip4Path := ipPath.ToIp4Path() assert.Equal(t, Ip4Path{ - Protocol: IpProtocolTcp, - SourceIp: [4]byte(net.IPv4(byte(72), byte(0), byte(0), byte(1)).To4()), - SourcePort: 40000 + 1, - DestinationIp: [4]byte(net.IPv4(byte(72), byte(1 + 1), byte(1 + 1), byte(1 + 1)).To4()), - DestinationPort: 443, + Protocol: IpProtocolTcp, + SourceIp: [4]byte(net.IPv4(byte(72), byte(0), byte(0), byte(1)).To4()), + SourcePort: 40000 + 1, + DestinationIp: [4]byte(net.IPv4(byte(72), byte(1+1), byte(1+1), byte(1+1)).To4()), + DestinationPort: 443, }, ip4Path) } - func TestUdp6Path(t *testing.T) { packet, _ := udp6Packet(1, 1, 1, 1) ipPath, err := ParseIpPath(packet) @@ -104,25 +97,24 @@ func TestUdp6Path(t *testing.T) { assert.Equal(t, IpProtocolUdp, ipPath.Protocol) assert.Equal(t, &IpPath{ - Version: 6, - Protocol: IpProtocolUdp, - SourceIp: net.IPv4(byte(72), byte(0), byte(0), byte(1)).To16(), - SourcePort: 40000 + 1, - DestinationIp: net.IPv4(byte(72), byte(1 + 1), byte(1 + 1), byte(1 + 1)).To16(), - DestinationPort: 443, + Version: 6, + Protocol: IpProtocolUdp, + SourceIp: net.IPv4(byte(72), byte(0), byte(0), byte(1)).To16(), + SourcePort: 40000 + 1, + DestinationIp: net.IPv4(byte(72), byte(1+1), byte(1+1), byte(1+1)).To16(), + DestinationPort: 443, }, ipPath) ip6Path := ipPath.ToIp6Path() assert.Equal(t, Ip6Path{ - Protocol: IpProtocolUdp, - SourceIp: [16]byte(net.IPv4(byte(72), byte(0), byte(0), byte(1)).To16()), - SourcePort: 40000 + 1, - DestinationIp: [16]byte(net.IPv4(byte(72), byte(1 + 1), byte(1 + 1), byte(1 + 1)).To16()), - DestinationPort: 443, + Protocol: IpProtocolUdp, + SourceIp: [16]byte(net.IPv4(byte(72), byte(0), byte(0), byte(1)).To16()), + SourcePort: 40000 + 1, + DestinationIp: [16]byte(net.IPv4(byte(72), byte(1+1), byte(1+1), byte(1+1)).To16()), + DestinationPort: 443, }, ip6Path) } - func TestTcp6Path(t *testing.T) { packet, _ := tcp6Packet(1, 1, 1, 1) ipPath, err := ParseIpPath(packet) @@ -131,203 +123,197 @@ func TestTcp6Path(t *testing.T) { assert.Equal(t, IpProtocolTcp, ipPath.Protocol) assert.Equal(t, &IpPath{ - Version: 6, - Protocol: IpProtocolTcp, - SourceIp: net.IPv4(byte(72), byte(0), byte(0), byte(1)).To16(), - SourcePort: 40000 + 1, - DestinationIp: net.IPv4(byte(72), byte(1 + 1), byte(1 + 1), byte(1 + 1)).To16(), - DestinationPort: 443, + Version: 6, + Protocol: IpProtocolTcp, + SourceIp: net.IPv4(byte(72), byte(0), byte(0), byte(1)).To16(), + SourcePort: 40000 + 1, + DestinationIp: net.IPv4(byte(72), byte(1+1), byte(1+1), byte(1+1)).To16(), + DestinationPort: 443, }, ipPath) ip6Path := ipPath.ToIp6Path() assert.Equal(t, Ip6Path{ - Protocol: IpProtocolTcp, - SourceIp: [16]byte(net.IPv4(byte(72), byte(0), byte(0), byte(1)).To16()), - SourcePort: 40000 + 1, - DestinationIp: [16]byte(net.IPv4(byte(72), byte(1 + 1), byte(1 + 1), byte(1 + 1)).To16()), - DestinationPort: 443, + Protocol: IpProtocolTcp, + SourceIp: [16]byte(net.IPv4(byte(72), byte(0), byte(0), byte(1)).To16()), + SourcePort: 40000 + 1, + DestinationIp: [16]byte(net.IPv4(byte(72), byte(1+1), byte(1+1), byte(1+1)).To16()), + DestinationPort: 443, }, ip6Path) } - - -func udp4Packet(s int, i int, j int, k int)(packet []byte, payload []byte) { +func udp4Packet(s int, i int, j int, k int) (packet []byte, payload []byte) { payload = make([]byte, 4) - binary.LittleEndian.PutUint32(payload, uint32(s)) + binary.LittleEndian.PutUint32(payload, uint32(s)) sourceIp := net.IPv4(72, 0, 0, 1) sourcePort := layers.UDPPort(40000 + s) - destinationIp := net.IPv4(byte(72), byte(1 + i), byte(1 + j), byte(1 + k)) + destinationIp := net.IPv4(byte(72), byte(1+i), byte(1+j), byte(1+k)) destinationPort := layers.UDPPort(443) - ip := &layers.IPv4{ - Version: 4, - TTL: 64, - SrcIP: sourceIp, - DstIP: destinationIp, - Protocol: layers.IPProtocolUDP, - } - - udp := layers.UDP{ - SrcPort: sourcePort, - DstPort: destinationPort, - } - udp.SetNetworkLayerForChecksum(ip) - - options := gopacket.SerializeOptions{ - ComputeChecksums: true, - FixLengths: true, - } - - buffer := gopacket.NewSerializeBufferExpectedSize(1024, 0) - err := gopacket.SerializeLayers(buffer, options, - gopacket.SerializableLayer(ip), - &udp, - gopacket.Payload(payload), - ) - if err != nil { - panic(err) - } - packet = buffer.Bytes() - - return -} + ip := &layers.IPv4{ + Version: 4, + TTL: 64, + SrcIP: sourceIp, + DstIP: destinationIp, + Protocol: layers.IPProtocolUDP, + } + + udp := layers.UDP{ + SrcPort: sourcePort, + DstPort: destinationPort, + } + udp.SetNetworkLayerForChecksum(ip) + options := gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + } -func tcp4Packet(s int, i int, j int, k int)(packet []byte, payload []byte) { + buffer := gopacket.NewSerializeBufferExpectedSize(1024, 0) + err := gopacket.SerializeLayers(buffer, options, + gopacket.SerializableLayer(ip), + &udp, + gopacket.Payload(payload), + ) + if err != nil { + panic(err) + } + packet = buffer.Bytes() + + return +} + +func tcp4Packet(s int, i int, j int, k int) (packet []byte, payload []byte) { payload = make([]byte, 4) - binary.LittleEndian.PutUint32(payload, uint32(s)) + binary.LittleEndian.PutUint32(payload, uint32(s)) sourceIp := net.IPv4(72, 0, 0, 1) sourcePort := layers.TCPPort(40000 + s) - destinationIp := net.IPv4(byte(72), byte(1 + i), byte(1 + j), byte(1 + k)) + destinationIp := net.IPv4(byte(72), byte(1+i), byte(1+j), byte(1+k)) destinationPort := layers.TCPPort(443) - ip := &layers.IPv4{ - Version: 4, - TTL: 64, - SrcIP: sourceIp, - DstIP: destinationIp, - Protocol: layers.IPProtocolTCP, - } - - tcp := layers.TCP{ - SrcPort: sourcePort, - DstPort: destinationPort, - Seq: 0, - Ack: 0, - Window: 1024, - } - tcp.SetNetworkLayerForChecksum(ip) - - options := gopacket.SerializeOptions{ - ComputeChecksums: true, - FixLengths: true, - } - - buffer := gopacket.NewSerializeBufferExpectedSize(1024, 0) - err := gopacket.SerializeLayers(buffer, options, - gopacket.SerializableLayer(ip), - &tcp, - gopacket.Payload(payload), - ) - if err != nil { - panic(err) - } - packet = buffer.Bytes() - - return -} + ip := &layers.IPv4{ + Version: 4, + TTL: 64, + SrcIP: sourceIp, + DstIP: destinationIp, + Protocol: layers.IPProtocolTCP, + } + + tcp := layers.TCP{ + SrcPort: sourcePort, + DstPort: destinationPort, + Seq: 0, + Ack: 0, + Window: 1024, + } + tcp.SetNetworkLayerForChecksum(ip) + + options := gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + } + + buffer := gopacket.NewSerializeBufferExpectedSize(1024, 0) + err := gopacket.SerializeLayers(buffer, options, + gopacket.SerializableLayer(ip), + &tcp, + gopacket.Payload(payload), + ) + if err != nil { + panic(err) + } + packet = buffer.Bytes() + return +} -func udp6Packet(s int, i int, j int, k int)(packet []byte, payload []byte) { +func udp6Packet(s int, i int, j int, k int) (packet []byte, payload []byte) { payload = make([]byte, 4) - binary.LittleEndian.PutUint32(payload, uint32(s)) + binary.LittleEndian.PutUint32(payload, uint32(s)) sourceIp := net.IPv4(72, 0, 0, 1).To16() sourcePort := layers.UDPPort(40000 + s) - destinationIp := net.IPv4(byte(72), byte(1 + i), byte(1 + j), byte(1 + k)).To16() + destinationIp := net.IPv4(byte(72), byte(1+i), byte(1+j), byte(1+k)).To16() destinationPort := layers.UDPPort(443) - ip := &layers.IPv6{ - Version: 6, - HopLimit: 64, - SrcIP: sourceIp, - DstIP: destinationIp, - NextHeader: layers.IPProtocolUDP, - } - - udp := layers.UDP{ - SrcPort: sourcePort, - DstPort: destinationPort, - } - udp.SetNetworkLayerForChecksum(ip) - - options := gopacket.SerializeOptions{ - ComputeChecksums: true, - FixLengths: true, - } - - buffer := gopacket.NewSerializeBufferExpectedSize(1024, 0) - err := gopacket.SerializeLayers(buffer, options, - gopacket.SerializableLayer(ip), - &udp, - gopacket.Payload(payload), - ) - if err != nil { - panic(err) - } - packet = buffer.Bytes() - - return -} + ip := &layers.IPv6{ + Version: 6, + HopLimit: 64, + SrcIP: sourceIp, + DstIP: destinationIp, + NextHeader: layers.IPProtocolUDP, + } + + udp := layers.UDP{ + SrcPort: sourcePort, + DstPort: destinationPort, + } + udp.SetNetworkLayerForChecksum(ip) + options := gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + } -func tcp6Packet(s int, i int, j int, k int)(packet []byte, payload []byte) { + buffer := gopacket.NewSerializeBufferExpectedSize(1024, 0) + err := gopacket.SerializeLayers(buffer, options, + gopacket.SerializableLayer(ip), + &udp, + gopacket.Payload(payload), + ) + if err != nil { + panic(err) + } + packet = buffer.Bytes() + + return +} + +func tcp6Packet(s int, i int, j int, k int) (packet []byte, payload []byte) { payload = make([]byte, 4) - binary.LittleEndian.PutUint32(payload, uint32(s)) + binary.LittleEndian.PutUint32(payload, uint32(s)) sourceIp := net.IPv4(72, 0, 0, 1).To16() sourcePort := layers.TCPPort(40000 + s) - destinationIp := net.IPv4(byte(72), byte(1 + i), byte(1 + j), byte(1 + k)).To16() + destinationIp := net.IPv4(byte(72), byte(1+i), byte(1+j), byte(1+k)).To16() destinationPort := layers.TCPPort(443) - ip := &layers.IPv6{ - Version: 6, - HopLimit: 64, - SrcIP: sourceIp, - DstIP: destinationIp, - NextHeader: layers.IPProtocolTCP, - } - - tcp := layers.TCP{ - SrcPort: sourcePort, - DstPort: destinationPort, - Seq: 0, - Ack: 0, - Window: 1024, - } - tcp.SetNetworkLayerForChecksum(ip) - - options := gopacket.SerializeOptions{ - ComputeChecksums: true, - FixLengths: true, - } - - buffer := gopacket.NewSerializeBufferExpectedSize(1024, 0) - err := gopacket.SerializeLayers(buffer, options, - gopacket.SerializableLayer(ip), - &tcp, - gopacket.Payload(payload), - ) - if err != nil { - panic(err) - } - packet = buffer.Bytes() - - return -} + ip := &layers.IPv6{ + Version: 6, + HopLimit: 64, + SrcIP: sourceIp, + DstIP: destinationIp, + NextHeader: layers.IPProtocolTCP, + } + + tcp := layers.TCP{ + SrcPort: sourcePort, + DstPort: destinationPort, + Seq: 0, + Ack: 0, + Window: 1024, + } + tcp.SetNetworkLayerForChecksum(ip) + + options := gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + } + + buffer := gopacket.NewSerializeBufferExpectedSize(1024, 0) + err := gopacket.SerializeLayers(buffer, options, + gopacket.SerializableLayer(ip), + &tcp, + gopacket.Payload(payload), + ) + if err != nil { + panic(err) + } + packet = buffer.Bytes() + return +} func testingNewClient(ctx context.Context, providerClient *Client, receivePacketCallback ReceivePacketFunction) (UserNatClient, error) { client := NewClientWithDefaults(ctx, NewId(), NewNoContractClientOob()) @@ -363,18 +349,16 @@ func testingNewClient(ctx context.Context, providerClient *Client, receivePacket ) } - // test with all sequence buffer sizes set to 0 func testClient[P comparable]( t *testing.T, - userNatClientGenerator func(context.Context, *Client, ReceivePacketFunction)(UserNatClient, error), + userNatClientGenerator func(context.Context, *Client, ReceivePacketFunction) (UserNatClient, error), packetGenerator PacketGeneratorFunction, - toComparableIpPath func(*IpPath)(P), + toComparableIpPath func(*IpPath) P, ) { // runs a send-receive test on the `UserNatClient` produced by `userNatClientGenerator` // this is a multi-threaded stress test that is meant to stress the buffers and routing - // n destinations // all have the same receiver callback, put into a channel of messages // echo the received packet @@ -384,7 +368,6 @@ func testClient[P comparable]( // make sure all packets are received // make sure all packets are echoed back - timeout := 30 * time.Second m := 6 @@ -393,7 +376,6 @@ func testClient[P comparable]( parallelCount := 6 echoCount := 2 - // each packet gets echoed back totalCount := parallelCount * m * n * n * n * repeatCount * (1 + echoCount) @@ -401,14 +383,12 @@ func testClient[P comparable]( // cSendCount := 0 // cReceiveCount := 0 - ctx, cancel := context.WithCancel(context.Background()) defer cancel() clientId := NewId() providerClientId := NewId() - settings := DefaultClientSettings() settings.SendBufferSettings.SequenceBufferSize = 0 settings.SendBufferSettings.AckBufferSize = 0 @@ -418,7 +398,6 @@ func testClient[P comparable]( providerClient := NewClient(ctx, providerClientId, NewNoContractClientOob(), settings) defer providerClient.Cancel() - type receivePacket struct { source TransferPath packet []byte @@ -426,7 +405,6 @@ func testClient[P comparable]( receivePackets := make(chan *receivePacket) - receivePacketCallback := func(source TransferPath, ipProtocol IpProtocol, packet []byte) { // record the echo packet @@ -435,19 +413,17 @@ func testClient[P comparable]( // // fmt.Printf("C Receive %d/%d (%.2f%%)\n", cReceiveCount, totalCount, 100.0 * float32(cReceiveCount) / float32(totalCount)) // cMutex.Unlock() - receivePacket := &receivePacket{ - source: source, - packet: packet, - } + receivePacket := &receivePacket{ + source: source, + packet: packet, + } - receivePackets <- receivePacket + receivePackets <- receivePacket } - natClient, err := userNatClientGenerator(ctx, providerClient, receivePacketCallback) assert.Equal(t, err, nil) - providerClient.AddReceiveCallback(func(source TransferPath, frames []*protocol.Frame, provideMode protocol.ProvideMode) { // cMutex.Lock() // cReceiveCount += 1 @@ -455,48 +431,47 @@ func testClient[P comparable]( // cMutex.Unlock() echo := func(packet []byte) { - ipPacketFromProvider := &protocol.IpPacketFromProvider{ - IpPacket: &protocol.IpPacket{ - PacketBytes: packet, - }, - } - frame, err := ToFrame(ipPacketFromProvider) - if err != nil { - panic(err) - } - - success := providerClient.SendWithTimeout(frame, source.Reverse(), func(err error) {}, -1) - assert.Equal(t, true, success) - - // cMutex.Lock() + ipPacketFromProvider := &protocol.IpPacketFromProvider{ + IpPacket: &protocol.IpPacket{ + PacketBytes: packet, + }, + } + frame, err := ToFrame(ipPacketFromProvider) + if err != nil { + panic(err) + } + + success := providerClient.SendWithTimeout(frame, source.Reverse(), func(err error) {}, -1) + assert.Equal(t, true, success) + + // cMutex.Lock() // cSendCount += 1 // // fmt.Printf("C Send %d/%d (%.2f%%)\n", cSendCount, totalCount, 100.0 * float32(cSendCount) / float32(totalCount)) // cMutex.Unlock() } for _, frame := range frames { - if ipPacketToProvider_, err := FromFrame(frame); err == nil { - if ipPacketToProvider, ok := ipPacketToProvider_.(*protocol.IpPacketToProvider); ok { - packet := ipPacketToProvider.IpPacket.PacketBytes - - receivePacket := &receivePacket{ - source: source, - packet: packet, - } - - receivePackets <- receivePacket - - for i := 0; i < echoCount; i += 1 { - // do not make a blocking call back into the client from the receiver - // this could deadlock the client depending on whether other messages are - // queued to this receiver - go echo(packet) - } - } - } - - } - }) + if ipPacketToProvider_, err := FromFrame(frame); err == nil { + if ipPacketToProvider, ok := ipPacketToProvider_.(*protocol.IpPacketToProvider); ok { + packet := ipPacketToProvider.IpPacket.PacketBytes + + receivePacket := &receivePacket{ + source: source, + packet: packet, + } + + receivePackets <- receivePacket + for i := 0; i < echoCount; i += 1 { + // do not make a blocking call back into the client from the receiver + // this could deadlock the client depending on whether other messages are + // queued to this receiver + go echo(packet) + } + } + } + + } + }) for p := 0; p < parallelCount; p += 1 { go func() { @@ -525,14 +500,13 @@ func testClient[P comparable]( }() } - comparableIpPathPayloads := map[P][][]byte{} comparableIpPathSources := map[P]map[TransferPath]bool{} for i := 0; i < totalCount; i += 1 { - fmt.Printf("[testr]%d/%d (%.2f%%)\n", i, totalCount, 100 * float32(i) / float32(totalCount)) + fmt.Printf("[testr]%d/%d (%.2f%%)\n", i, totalCount, 100*float32(i)/float32(totalCount)) select { - case receivePacket := <- receivePackets: + case receivePacket := <-receivePackets: // fmt.Printf("Receive %d/%d (%.2f%%)\n", i + 1, totalCount, 100.0 * float32(i + 1) / float32(totalCount)) ipPath, err := ParseIpPath(receivePacket.packet) @@ -542,23 +516,23 @@ func testClient[P comparable]( switch ipPath.Version { case 4: ipv4 := layers.IPv4{} - ipv4.DecodeFromBytes(receivePacket.packet, gopacket.NilDecodeFeedback) - payload = ipv4.Payload + ipv4.DecodeFromBytes(receivePacket.packet, gopacket.NilDecodeFeedback) + payload = ipv4.Payload case 6: ipv6 := layers.IPv6{} - ipv6.DecodeFromBytes(receivePacket.packet, gopacket.NilDecodeFeedback) - payload = ipv6.Payload - } + ipv6.DecodeFromBytes(receivePacket.packet, gopacket.NilDecodeFeedback) + payload = ipv6.Payload + } switch ipPath.Protocol { case IpProtocolUdp: udp := layers.UDP{} - udp.DecodeFromBytes(payload, gopacket.NilDecodeFeedback) - payload = udp.Payload + udp.DecodeFromBytes(payload, gopacket.NilDecodeFeedback) + payload = udp.Payload case IpProtocolTcp: tcp := layers.TCP{} - tcp.DecodeFromBytes(payload, gopacket.NilDecodeFeedback) - payload = tcp.Payload + tcp.DecodeFromBytes(payload, gopacket.NilDecodeFeedback) + payload = tcp.Payload } comparableIpPath := toComparableIpPath(ipPath) @@ -570,7 +544,7 @@ func testClient[P comparable]( comparableIpPathSources[comparableIpPath] = sources } sources[receivePacket.source] = true - case <- time.After(timeout): + case <-time.After(timeout): t.FailNow() } } @@ -597,7 +571,7 @@ func testClient[P comparable]( } } - assert.Equal(t, parallelCount * repeatCount * (1 + echoCount), count) + assert.Equal(t, parallelCount*repeatCount*(1+echoCount), count) sources := comparableIpPathSources[comparableIpPath] @@ -611,5 +585,3 @@ func testClient[P comparable]( } } } - - diff --git a/connect/transfer.go b/connect/transfer.go index 3b6b467..5081782 100644 --- a/connect/transfer.go +++ b/connect/transfer.go @@ -2,11 +2,12 @@ package connect import ( "context" - "time" - "sync" "errors" - "math" "fmt" + "math" + "sync" + "time" + // "runtime/debug" // "runtime" // "reflect" @@ -18,10 +19,9 @@ import ( "github.com/golang/glog" - "bringyour.com/protocol" + "github.com/bringyour/connect/protocol" ) - /* Sends frames to destinations with properties: - as long the sending client is active, frames are eventually delivered up to timeout @@ -42,133 +42,120 @@ Each transport should apply the forwarding ACL: */ - // The transfer speed of each client is limited by its slowest destination. // All traffic is multiplexed to a single connection, and blocking // the connection ultimately limits the rate of `SendWithTimeout`. // In this a client is similar to a socket. Multiple clients // can be active in parallel, each limited by their slowest destination. - // use 0 for deadlock testing const DefaultTransferBufferSize = 32 - const VerifyForwardMessages = false - type AckFunction = func(err error) + // provideMode is the mode of where these frames are from: network, friends and family, public // provideMode nil means no contract type ReceiveFunction = func(source TransferPath, frames []*protocol.Frame, provideMode protocol.ProvideMode) type ForwardFunction = func(path TransferPath, transferFrameBytes []byte) - func DefaultClientSettings() *ClientSettings { return &ClientSettings{ - SendBufferSize: DefaultTransferBufferSize, - ForwardBufferSize: DefaultTransferBufferSize, - ReadTimeout: 30 * time.Second, - BufferTimeout: 30 * time.Second, - ControlWriteTimeout: 30 * time.Second, - SendBufferSettings: DefaultSendBufferSettings(), - ReceiveBufferSettings: DefaultReceiveBufferSettings(), - ForwardBufferSettings: DefaultForwardBufferSettings(), + SendBufferSize: DefaultTransferBufferSize, + ForwardBufferSize: DefaultTransferBufferSize, + ReadTimeout: 30 * time.Second, + BufferTimeout: 30 * time.Second, + ControlWriteTimeout: 30 * time.Second, + SendBufferSettings: DefaultSendBufferSettings(), + ReceiveBufferSettings: DefaultReceiveBufferSettings(), + ForwardBufferSettings: DefaultForwardBufferSettings(), ContractManagerSettings: DefaultContractManagerSettings(), - StreamManagerSettings: DefaultStreamManagerSettings(), + StreamManagerSettings: DefaultStreamManagerSettings(), } } - func DefaultClientSettingsNoNetworkEvents() *ClientSettings { clientSettings := DefaultClientSettings() clientSettings.ContractManagerSettings = DefaultContractManagerSettingsNoNetworkEvents() return clientSettings } - func DefaultSendBufferSettings() *SendBufferSettings { return &SendBufferSettings{ - CreateContractTimeout: 30 * time.Second, + CreateContractTimeout: 30 * time.Second, CreateContractRetryInterval: 5 * time.Second, // this should be greater than the rtt under load // TODO use an rtt estimator based on the ack times ResendInterval: 1 * time.Second, // no backoff ResendBackoffScale: 0, - AckTimeout: 60 * time.Second, - IdleTimeout: 60 * time.Second, + AckTimeout: 60 * time.Second, + IdleTimeout: 60 * time.Second, // pause on resend for selectively acked messaged SelectiveAckTimeout: 5 * time.Second, - SequenceBufferSize: DefaultTransferBufferSize, - AckBufferSize: DefaultTransferBufferSize, + SequenceBufferSize: DefaultTransferBufferSize, + AckBufferSize: DefaultTransferBufferSize, MinMessageByteCount: ByteCount(1), // this includes transport reconnections - WriteTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, ResendQueueMaxByteCount: mib(1), - ContractFillFraction: 0.5, + ContractFillFraction: 0.5, } } - func DefaultReceiveBufferSettings() *ReceiveBufferSettings { - return &ReceiveBufferSettings { + return &ReceiveBufferSettings{ GapTimeout: 60 * time.Second, // the receive idle timeout should be a bit longer than the send idle timeout - IdleTimeout: 120 * time.Second, + IdleTimeout: 120 * time.Second, SequenceBufferSize: DefaultTransferBufferSize, // AckBufferSize: DefaultTransferBufferSize, - AckCompressTimeout: 10 * time.Millisecond, - MinMessageByteCount: ByteCount(1), + AckCompressTimeout: 10 * time.Millisecond, + MinMessageByteCount: ByteCount(1), ResendAbuseThreshold: 4, - ResendAbuseMultiple: 0.5, + ResendAbuseMultiple: 0.5, MaxPeerAuditDuration: 60 * time.Second, // this includes transport reconnections - WriteTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, ReceiveQueueMaxByteCount: mib(2), } } - func DefaultForwardBufferSettings() *ForwardBufferSettings { - return &ForwardBufferSettings { - IdleTimeout: 300 * time.Second, + return &ForwardBufferSettings{ + IdleTimeout: 300 * time.Second, SequenceBufferSize: DefaultTransferBufferSize, - WriteTimeout: 1 * time.Second, + WriteTimeout: 1 * time.Second, } } - type SendPack struct { TransferOptions // frame and destination is repacked by the send buffer into a Pack, // with destination and frame from the tframe, and other pack properties filled in by the buffer - Frame *protocol.Frame - Destination TransferPath + Frame *protocol.Frame + Destination TransferPath IntermediaryIds MultiHopId // called (true) when the pack is ack'd, or (false) if not ack'd (closed before ack) - AckCallback AckFunction + AckCallback AckFunction MessageByteCount ByteCount } - type ReceivePack struct { - Source TransferPath - SequenceId Id - Pack *protocol.Pack - ReceiveCallback ReceiveFunction + Source TransferPath + SequenceId Id + Pack *protocol.Pack + ReceiveCallback ReceiveFunction MessageByteCount ByteCount } - type ForwardPack struct { - Destination TransferPath + Destination TransferPath TransferFrameBytes []byte } - - type TransferOptions struct { // items can choose to not be acked // in this case, the ack callback is called on send, and no retry is done @@ -184,13 +171,12 @@ type TransferOptions struct { func DefaultTransferOpts() TransferOptions { return TransferOptions{ - Ack: true, + Ack: true, CompanionContract: false, - ForceStream: false, + ForceStream: false, } } - type transferOptionsSetAck struct { Ack bool } @@ -201,7 +187,6 @@ func NoAck() transferOptionsSetAck { } } - type transferOptionsSetCompanionContract struct { CompanionContract bool } @@ -212,7 +197,6 @@ func CompanionContract() transferOptionsSetCompanionContract { } } - type transferOptionsSetForceStream struct { ForceStream bool } @@ -223,29 +207,26 @@ func ForceStream() transferOptionsSetForceStream { } } - - type ClientSettings struct { - SendBufferSize int - ForwardBufferSize int - ReadTimeout time.Duration - BufferTimeout time.Duration + SendBufferSize int + ForwardBufferSize int + ReadTimeout time.Duration + BufferTimeout time.Duration ControlWriteTimeout time.Duration - SendBufferSettings *SendBufferSettings - ReceiveBufferSettings *ReceiveBufferSettings - ForwardBufferSettings *ForwardBufferSettings + SendBufferSettings *SendBufferSettings + ReceiveBufferSettings *ReceiveBufferSettings + ForwardBufferSettings *ForwardBufferSettings ContractManagerSettings *ContractManagerSettings - StreamManagerSettings *StreamManagerSettings + StreamManagerSettings *StreamManagerSettings } - // note all callbacks are wrapped to check for nil and recover from errors type Client struct { - ctx context.Context + ctx context.Context cancel context.CancelFunc - clientId Id + clientId Id clientTag string clientOob OutOfBandControl @@ -256,15 +237,15 @@ type Client struct { loopback chan *SendPack - routeManager *RouteManager + routeManager *RouteManager contractManager *ContractManager - streamManager *StreamManager - sendBuffer *SendBuffer - receiveBuffer *ReceiveBuffer - forwardBuffer *ForwardBuffer + streamManager *StreamManager + sendBuffer *SendBuffer + receiveBuffer *ReceiveBuffer + forwardBuffer *ForwardBuffer contractManagerUnsub func() - streamManagerUnsub func() + streamManagerUnsub func() } func NewClientWithDefaults( @@ -299,15 +280,15 @@ func NewClientWithTag( ) *Client { cancelCtx, cancel := context.WithCancel(ctx) client := &Client{ - ctx: cancelCtx, - cancel: cancel, - clientId: clientId, - clientTag: clientTag, - clientOob: clientOob, - settings: settings, + ctx: cancelCtx, + cancel: cancel, + clientId: clientId, + clientTag: clientTag, + clientOob: clientOob, + settings: settings, receiveCallbacks: NewCallbackList[ReceiveFunction](), forwardCallbacks: NewCallbackList[ForwardFunction](), - loopback: make(chan *SendPack), + loopback: make(chan *SendPack), } routeManager := NewRouteManager(ctx, clientTag) @@ -359,7 +340,7 @@ func (self *Client) ClientOob() OutOfBandControl { func (self *Client) ReportAbuse(source TransferPath) { peerAudit := NewSequencePeerAudit(self, source, 0) - peerAudit.Update(func (peerAudit *PeerAudit) { + peerAudit.Update(func(peerAudit *PeerAudit) { peerAudit.Abuse = true }) peerAudit.Complete() @@ -372,7 +353,7 @@ func (self *Client) ForwardWithTimeout(transferFrameBytes []byte, timeout time.D func (self *Client) ForwardWithTimeoutDetailed(transferFrameBytes []byte, timeout time.Duration) (bool, error) { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done") default: } @@ -392,7 +373,7 @@ func (self *Client) ForwardWithTimeoutDetailed(transferFrameBytes []byte, timeou destination := path.DestinationMask() forwardPack := &ForwardPack{ - Destination: destination, + Destination: destination, TransferFrameBytes: transferFrameBytes, } @@ -490,7 +471,7 @@ func (self *Client) sendWithTimeoutDetailed( } select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done") default: } @@ -519,11 +500,11 @@ func (self *Client) sendWithTimeoutDetailed( messageByteCount := ByteCount(len(frame.MessageBytes)) sendPack := &SendPack{ - TransferOptions: transferOpts, - Frame: frame, - Destination: destination, - IntermediaryIds: intermediaryIds, - AckCallback: safeAckCallback, + TransferOptions: transferOpts, + Frame: frame, + Destination: destination, + IntermediaryIds: intermediaryIds, + AckCallback: safeAckCallback, MessageByteCount: messageByteCount, } @@ -531,14 +512,14 @@ func (self *Client) sendWithTimeoutDetailed( // loopback if timeout < 0 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done") case self.loopback <- sendPack: return true, nil } } else if timeout == 0 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done") case self.loopback <- sendPack: return true, nil @@ -547,11 +528,11 @@ func (self *Client) sendWithTimeoutDetailed( } } else { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done") case self.loopback <- sendPack: return true, nil - case <- time.After(timeout): + case <-time.After(timeout): return false, nil } } @@ -588,7 +569,7 @@ func (self *Client) SendMultiHop(frame *protocol.Frame, destination MultiHopId, // ReceiveFunction func (self *Client) receive(source TransferPath, frames []*protocol.Frame, provideMode protocol.ProvideMode) { for _, receiveCallback := range self.receiveCallbacks.Get() { - c := func()(any) { + c := func() any { return HandleError(func() { receiveCallback(source, frames, provideMode) }) @@ -607,7 +588,7 @@ func (self *Client) receive(source TransferPath, frames []*protocol.Frame, provi // ForwardFunction func (self *Client) forward(path TransferPath, transferFrameBytes []byte) { for _, forwardCallback := range self.forwardCallbacks.Get() { - c := func()(any) { + c := func() any { return HandleError(func() { forwardCallback(path, transferFrameBytes) }) @@ -639,7 +620,7 @@ func (self *Client) AddForwardCallback(forwardCallback ForwardFunction) func() { func (self *Client) run() { defer self.cancel() - + // receive multiRouteReader := self.routeManager.OpenMultiRouteReader(DestinationId(self.clientId)) defer self.routeManager.CloseMultiRouteReader(multiRouteReader) @@ -655,9 +636,9 @@ func (self *Client) run() { go func() { for { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return - case sendPack := <- self.loopback: + case sendPack := <-self.loopback: HandleError(func() { self.receive( SourceId(self.clientId), @@ -674,14 +655,14 @@ func (self *Client) run() { for { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return default: } var transferFrameBytes []byte var err error - c := func()(error) { + c := func() error { transferFrameBytes, err = multiRouteReader.Read(self.ctx, self.settings.ReadTimeout) return err } @@ -748,7 +729,7 @@ func (self *Client) run() { }) continue } - c := func()(bool) { + c := func() bool { return self.sendBuffer.Ack( source.Reverse(), ack, @@ -778,12 +759,12 @@ func (self *Client) run() { continue } messageByteCount := MessageByteCount(pack.Frames) - c := func()(bool) { + c := func() bool { success, err := self.receiveBuffer.Pack(&ReceivePack{ - Source: source, - SequenceId: sequenceId, - Pack: pack, - ReceiveCallback: self.receive, + Source: source, + SequenceId: sequenceId, + Pack: pack, + ReceiveCallback: self.receive, MessageByteCount: messageByteCount, }, self.settings.BufferTimeout) return success && err == nil @@ -875,7 +856,7 @@ func (self *Client) ReceiveQueueSize(source TransferPath, sequenceId Id) (int, B func (self *Client) IsDone() bool { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return true default: return false @@ -918,24 +899,23 @@ func (self *Client) Flush() { self.contractManager.Flush(false) } - type SendBufferSettings struct { - CreateContractTimeout time.Duration + CreateContractTimeout time.Duration CreateContractRetryInterval time.Duration // TODO replace this with round trip time estimation // resend timeout is the initial time between successive send attempts. Does linear backoff - ResendInterval time.Duration + ResendInterval time.Duration ResendBackoffScale float64 // on ack timeout, no longer attempt to retransmit and notify of ack failure - AckTimeout time.Duration + AckTimeout time.Duration IdleTimeout time.Duration SelectiveAckTimeout time.Duration SequenceBufferSize int - AckBufferSize int + AckBufferSize int MinMessageByteCount ByteCount @@ -948,46 +928,46 @@ type SendBufferSettings struct { } type sendSequenceId struct { - Destination TransferPath - IntermediaryIds MultiHopId + Destination TransferPath + IntermediaryIds MultiHopId CompanionContract bool - ForceStream bool + ForceStream bool } type SendBuffer struct { - ctx context.Context + ctx context.Context client *Client - + sendBufferSettings *SendBufferSettings - mutex sync.Mutex - sendSequences map[sendSequenceId]*SendSequence + mutex sync.Mutex + sendSequences map[sendSequenceId]*SendSequence sendSequencesByDestination map[TransferPath]map[*SendSequence]bool - sendSequenceDestinations map[*SendSequence]map[TransferPath]bool + sendSequenceDestinations map[*SendSequence]map[TransferPath]bool } func NewSendBuffer(ctx context.Context, - client *Client, - sendBufferSettings *SendBufferSettings) *SendBuffer { + client *Client, + sendBufferSettings *SendBufferSettings) *SendBuffer { return &SendBuffer{ - ctx: ctx, - client: client, - sendBufferSettings: sendBufferSettings, - sendSequences: map[sendSequenceId]*SendSequence{}, + ctx: ctx, + client: client, + sendBufferSettings: sendBufferSettings, + sendSequences: map[sendSequenceId]*SendSequence{}, sendSequencesByDestination: map[TransferPath]map[*SendSequence]bool{}, - sendSequenceDestinations: map[*SendSequence]map[TransferPath]bool{}, + sendSequenceDestinations: map[*SendSequence]map[TransferPath]bool{}, } } func (self *SendBuffer) Pack(sendPack *SendPack, timeout time.Duration) (bool, error) { sendSequenceId := sendSequenceId{ - Destination: sendPack.Destination, - IntermediaryIds: sendPack.IntermediaryIds, + Destination: sendPack.Destination, + IntermediaryIds: sendPack.IntermediaryIds, CompanionContract: sendPack.TransferOptions.CompanionContract, - ForceStream: sendPack.TransferOptions.ForceStream, + ForceStream: sendPack.TransferOptions.ForceStream, } - initSendSequence := func(skip *SendSequence)(*SendSequence) { + initSendSequence := func(skip *SendSequence) *SendSequence { self.mutex.Lock() defer self.mutex.Unlock() @@ -1045,7 +1025,7 @@ func (self *SendBuffer) Pack(sendPack *SendPack, timeout time.Duration) (bool, e var err error for i := 0; i < 2; i += 1 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") default: } @@ -1059,7 +1039,7 @@ func (self *SendBuffer) Pack(sendPack *SendPack, timeout time.Duration) (bool, e } func (self *SendBuffer) Ack(destination TransferPath, ack *protocol.Ack, timeout time.Duration) bool { - sendSequences := func()([]*SendSequence) { + sendSequences := func() []*SendSequence { self.mutex.Lock() defer self.mutex.Unlock() if sendSequences, ok := self.sendSequencesByDestination[destination]; ok { @@ -1068,7 +1048,7 @@ func (self *SendBuffer) Ack(destination TransferPath, ack *protocol.Ack, timeout return []*SendSequence{} } } - + anyFound := false anySuccess := false for _, seq := range sendSequences() { @@ -1085,14 +1065,14 @@ func (self *SendBuffer) Ack(destination TransferPath, ack *protocol.Ack, timeout } func (self *SendBuffer) ResendQueueSize(destination TransferPath, intermediaryIds MultiHopId, companionContract bool, forceStream bool) (int, ByteCount, Id) { - sendSequence := func()(*SendSequence) { + sendSequence := func() *SendSequence { self.mutex.Lock() defer self.mutex.Unlock() return self.sendSequences[sendSequenceId{ - Destination: destination, - IntermediaryIds: intermediaryIds, + Destination: destination, + IntermediaryIds: intermediaryIds, CompanionContract: companionContract, - ForceStream: forceStream, + ForceStream: forceStream, }] } @@ -1156,19 +1136,18 @@ func (self *SendBuffer) Flush() { } } - type SendSequence struct { - ctx context.Context + ctx context.Context cancel context.CancelFunc - client *Client + client *Client sendBuffer *SendBuffer - destination TransferPath - intermediaryIds MultiHopId + destination TransferPath + intermediaryIds MultiHopId companionContract bool - forceStream bool - sequenceId Id + forceStream bool + sequenceId Id sendBufferSettings *SendBufferSettings @@ -1179,50 +1158,50 @@ type SendSequence struct { openSendContracts map[Id]*sequenceContract packMutex sync.Mutex - packs chan *SendPack - ackMutex sync.Mutex - acks chan *protocol.Ack + packs chan *SendPack + ackMutex sync.Mutex + acks chan *protocol.Ack - resendQueue *resendQueue - sendItems []*sendItem + resendQueue *resendQueue + sendItems []*sendItem nextSequenceNumber uint64 idleCondition *IdleCondition - contractMultiRouteWriter MultiRouteWriter + contractMultiRouteWriter MultiRouteWriter contractMultiRouteWriterDestination TransferPath } func NewSendSequence( - ctx context.Context, - client *Client, - sendBuffer *SendBuffer, - destination TransferPath, - intermediaryIds MultiHopId, - companionContract bool, - forceStream bool, - sendBufferSettings *SendBufferSettings) *SendSequence { + ctx context.Context, + client *Client, + sendBuffer *SendBuffer, + destination TransferPath, + intermediaryIds MultiHopId, + companionContract bool, + forceStream bool, + sendBufferSettings *SendBufferSettings) *SendSequence { cancelCtx, cancel := context.WithCancel(ctx) return &SendSequence{ - ctx: cancelCtx, - cancel: cancel, - client: client, - sendBuffer: sendBuffer, - destination: destination, - intermediaryIds: intermediaryIds, - companionContract: companionContract, - forceStream: forceStream, - sequenceId: NewId(), + ctx: cancelCtx, + cancel: cancel, + client: client, + sendBuffer: sendBuffer, + destination: destination, + intermediaryIds: intermediaryIds, + companionContract: companionContract, + forceStream: forceStream, + sequenceId: NewId(), sendBufferSettings: sendBufferSettings, - sendContract: nil, - openSendContracts: map[Id]*sequenceContract{}, - packs: make(chan *SendPack, sendBufferSettings.SequenceBufferSize), - acks: make(chan *protocol.Ack, sendBufferSettings.AckBufferSize), - resendQueue: newResendQueue(), - sendItems: []*sendItem{}, + sendContract: nil, + openSendContracts: map[Id]*sequenceContract{}, + packs: make(chan *SendPack, sendBufferSettings.SequenceBufferSize), + acks: make(chan *protocol.Ack, sendBufferSettings.AckBufferSize), + resendQueue: newResendQueue(), + sendItems: []*sendItem{}, nextSequenceNumber: 0, - idleCondition: NewIdleCondition(), + idleCondition: NewIdleCondition(), } } @@ -1237,7 +1216,7 @@ func (self *SendSequence) Pack(sendPack *SendPack, timeout time.Duration) (bool, defer self.packMutex.Unlock() select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") default: } @@ -1249,14 +1228,14 @@ func (self *SendSequence) Pack(sendPack *SendPack, timeout time.Duration) (bool, if timeout < 0 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") case self.packs <- sendPack: return true, nil } } else if timeout == 0 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") case self.packs <- sendPack: return true, nil @@ -1265,11 +1244,11 @@ func (self *SendSequence) Pack(sendPack *SendPack, timeout time.Duration) (bool, } } else { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") case self.packs <- sendPack: return true, nil - case <- time.After(timeout): + case <-time.After(timeout): return false, nil } } @@ -1289,21 +1268,21 @@ func (self *SendSequence) Ack(ack *protocol.Ack, timeout time.Duration) (bool, e } select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") default: } if timeout < 0 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") case self.acks <- ack: return true, nil } } else if timeout == 0 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") case self.acks <- ack: return true, nil @@ -1312,11 +1291,11 @@ func (self *SendSequence) Ack(ack *protocol.Ack, timeout time.Duration) (bool, e } } else { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") case self.acks <- ack: return true, nil - case <- time.After(timeout): + case <-time.After(timeout): return false, nil } } @@ -1349,10 +1328,10 @@ func (self *SendSequence) Run() { // flush queued up contracts // remove used contract ids because all used contracts were closed above contractKey := ContractKey{ - Destination: self.destination, - IntermediaryIds: self.intermediaryIds, + Destination: self.destination, + IntermediaryIds: self.intermediaryIds, CompanionContract: self.companionContract, - ForceStream: self.forceStream, + ForceStream: self.forceStream, } self.client.ContractManager().FlushContractQueue(contractKey, true) @@ -1365,18 +1344,18 @@ func (self *SendSequence) Run() { for { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return - case ack, ok := <- self.acks: + case ack, ok := <-self.acks: if !ok { return } if messageId, err := IdFromBytes(ack.MessageId); err == nil { if sequenceNumber, ok := self.resendQueue.ContainsMessageId(messageId); ok { ack := &sequenceAck{ - messageId: messageId, + messageId: messageId, sequenceNumber: sequenceNumber, - selective: ack.Selective, + selective: ack.Selective, } ackWindow.Update(ack) } @@ -1395,11 +1374,10 @@ func (self *SendSequence) Run() { self.receiveAck(messageId, true) } - sendTime := time.Now() var timeout time.Duration - if self.resendQueue.Len() == 0 { + if self.resendQueue.Len() == 0 { timeout = self.sendBufferSettings.IdleTimeout } else { timeout = self.sendBufferSettings.AckTimeout @@ -1446,7 +1424,7 @@ func (self *SendSequence) Run() { transferFrameBytes = item.transferFrameBytes } - c := func()(error) { + c := func() error { return self.openContractMultiRouteWriter().Write( self.ctx, transferFrameBytes, @@ -1475,7 +1453,7 @@ func (self *SendSequence) Run() { item.sendCount += 1 // linear backoff // itemResendTimeout := self.sendBufferSettings.ResendInterval - itemResendTimeout := time.Duration(float64(self.sendBufferSettings.ResendInterval) * (1 + self.sendBufferSettings.ResendBackoffScale * float64(item.sendCount))) + itemResendTimeout := time.Duration(float64(self.sendBufferSettings.ResendInterval) * (1 + self.sendBufferSettings.ResendBackoffScale*float64(item.sendCount))) if itemResendTimeout < itemAckTimeout { item.resendTime = sendTime.Add(itemResendTimeout) } else { @@ -1486,38 +1464,38 @@ func (self *SendSequence) Run() { } checkpointId := self.idleCondition.Checkpoint() - + // approximate since this cannot consider the next message byte size - canQueue := func()(bool) { - // always allow at least one item in the resend queue - queueSize, queueByteCount := self.resendQueue.QueueSize() - if 0 == queueSize { - return true - } - return queueByteCount < self.sendBufferSettings.ResendQueueMaxByteCount + canQueue := func() bool { + // always allow at least one item in the resend queue + queueSize, queueByteCount := self.resendQueue.QueueSize() + if 0 == queueSize { + return true + } + return queueByteCount < self.sendBufferSettings.ResendQueueMaxByteCount } if !canQueue() { // wait for acks select { - case <- self.ctx.Done(): - return - case <- ackSnapshot.ackNotify: - case <- time.After(timeout): + case <-self.ctx.Done(): + return + case <-ackSnapshot.ackNotify: + case <-time.After(timeout): if 0 == self.resendQueue.Len() { // idle timeout if self.idleCondition.Close(checkpointId) { // close the sequence - return + return } // else there are pending updates } } } else { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return - case <- ackSnapshot.ackNotify: - case sendPack, ok := <- self.packs: + case <-ackSnapshot.ackNotify: + case sendPack, ok := <-self.packs: if !ok { return } @@ -1533,7 +1511,7 @@ func (self *SendSequence) Run() { sendPack.AckCallback(errors.New("No contract")) return } - case <- time.After(timeout): + case <-time.After(timeout): if 0 == self.resendQueue.Len() { // idle timeout if self.idleCondition.Close(checkpointId) { @@ -1548,7 +1526,7 @@ func (self *SendSequence) Run() { } func (self *SendSequence) updateContract(messageByteCount ByteCount) bool { - // `sendNoContract` is a mutual configuration + // `sendNoContract` is a mutual configuration // both sides must configure themselves to require no contract from each other isStream := self.destination.IsStream() || 0 < self.intermediaryIds.Len() if !isStream && self.client.ContractManager().SendNoContract(self.destination.DestinationId) { @@ -1558,20 +1536,19 @@ func (self *SendSequence) updateContract(messageByteCount ByteCount) bool { return true } - createContract := func()(bool) { + createContract := func() bool { // the max overhead of the pack frame // this is needed because the size of the contract pack is counted against the contract // maxContractMessageByteCount := ByteCount(256) effectiveContractTransferByteCount := ByteCount(float32(self.client.ContractManager().StandardContractTransferByteCount()) * self.sendBufferSettings.ContractFillFraction) - if effectiveContractTransferByteCount < messageByteCount + self.sendBufferSettings.MinMessageByteCount /*+ maxContractMessageByteCount*/ { + if effectiveContractTransferByteCount < messageByteCount+self.sendBufferSettings.MinMessageByteCount /*+ maxContractMessageByteCount*/ { // this pack does not fit into a standard contract // TODO allow requesting larger contracts panic("Message too large for contract. It can never be sent.") } - - setNextContract := func(contract *protocol.Contract)(bool) { + setNextContract := func(contract *protocol.Contract) bool { nextSendContract, err := newSequenceContract( "s", contract, @@ -1590,7 +1567,7 @@ func (self *SendSequence) updateContract(messageByteCount ByteCount) bool { self.setContract(nextSendContract) // append the contract to the sequence - self.sendWithSetContract(nil, func(error){}, true, true) + self.sendWithSetContract(nil, func(error) {}, true, true) return true } else { @@ -1603,12 +1580,12 @@ func (self *SendSequence) updateContract(messageByteCount ByteCount) bool { } } - nextContract := func(timeout time.Duration)(bool) { + nextContract := func(timeout time.Duration) bool { contractKey := ContractKey{ - Destination: self.destination, - IntermediaryIds: self.intermediaryIds, + Destination: self.destination, + IntermediaryIds: self.intermediaryIds, CompanionContract: self.companionContract, - ForceStream: self.forceStream, + ForceStream: self.forceStream, } if contract := self.client.ContractManager().TakeContract(self.ctx, contractKey, timeout); contract != nil && setNextContract(contract) { // async queue up the next contract @@ -1621,11 +1598,11 @@ func (self *SendSequence) updateContract(messageByteCount ByteCount) bool { return false } } - traceNextContract := func(timeout time.Duration)(bool) { + traceNextContract := func(timeout time.Duration) bool { if glog.V(2) { return TraceWithReturn( fmt.Sprintf("[s]%s->%s...%s s(%s) next contract", self.client.ClientTag(), self.intermediaryIds, self.destination.DestinationId, self.destination.StreamId), - func()(bool) { + func() bool { return nextContract(timeout) }, ) @@ -1641,7 +1618,7 @@ func (self *SendSequence) updateContract(messageByteCount ByteCount) bool { endTime := time.Now().Add(self.sendBufferSettings.CreateContractTimeout) for { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false default: } @@ -1653,10 +1630,10 @@ func (self *SendSequence) updateContract(messageByteCount ByteCount) bool { // async queue up the next contract contractKey := ContractKey{ - Destination: self.destination, - IntermediaryIds: self.intermediaryIds, + Destination: self.destination, + IntermediaryIds: self.intermediaryIds, CompanionContract: self.companionContract, - ForceStream: self.forceStream, + ForceStream: self.forceStream, } self.client.ContractManager().CreateContract( contractKey, @@ -1714,7 +1691,7 @@ func (self *SendSequence) sendWithSetContract( ) { sendTime := time.Now() messageId := NewId() - + var contractId *Id if self.sendContract != nil { contractId = &self.sendContract.contractId @@ -1734,8 +1711,8 @@ func (self *SendSequence) sendWithSetContract( var contractFrame *protocol.Frame if (head || setContract) && self.sendContract != nil { contractMessageBytes, _ := proto.Marshal(self.sendContract.contract) - contractFrame = &protocol.Frame{ - MessageType: protocol.MessageType_TransferContract, + contractFrame = &protocol.Frame{ + MessageType: protocol.MessageType_TransferContract, MessageBytes: contractMessageBytes, } } @@ -1746,13 +1723,13 @@ func (self *SendSequence) sendWithSetContract( } pack := &protocol.Pack{ - MessageId: messageId.Bytes(), - SequenceId: self.sequenceId.Bytes(), + MessageId: messageId.Bytes(), + SequenceId: self.sequenceId.Bytes(), SequenceNumber: sequenceNumber, - Head: head, - Frames: frames, - ContractFrame: contractFrame, - Nack: !ack, + Head: head, + Frames: frames, + ContractFrame: contractFrame, + Nack: !ack, } packBytes, _ := proto.Marshal(pack) @@ -1766,7 +1743,7 @@ func (self *SendSequence) sendWithSetContract( transferFrame := &protocol.TransferFrame{ TransferPath: path.ToProtobuf(), Frame: &protocol.Frame{ - MessageType: protocol.MessageType_TransferPack, + MessageType: protocol.MessageType_TransferPack, MessageBytes: packBytes, }, } @@ -1777,21 +1754,21 @@ func (self *SendSequence) sendWithSetContract( item := &sendItem{ transferItem: transferItem{ - messageId: messageId, - sequenceNumber: sequenceNumber, + messageId: messageId, + sequenceNumber: sequenceNumber, messageByteCount: messageByteCount, }, - contractId: contractId, - sendTime: sendTime, - resendTime: sendTime.Add(self.sendBufferSettings.ResendInterval), - sendCount: 1, - head: head, - hasContractFrame: (contractFrame != nil), + contractId: contractId, + sendTime: sendTime, + resendTime: sendTime.Add(self.sendBufferSettings.ResendInterval), + sendCount: 1, + head: head, + hasContractFrame: (contractFrame != nil), transferFrameBytes: transferFrameBytes, - ackCallback: ackCallback, + ackCallback: ackCallback, } - c := func()(error) { + c := func() error { return self.openContractMultiRouteWriter().Write( self.ctx, item.transferFrameBytes, @@ -1845,8 +1822,8 @@ func (self *SendSequence) setHead(item *sendItem) ([]byte, error) { if item.contractId != nil && !item.hasContractFrame { sendContract := self.openSendContracts[*item.contractId] contractMessageBytes, _ := proto.Marshal(sendContract.contract) - pack.ContractFrame = &protocol.Frame{ - MessageType: protocol.MessageType_TransferContract, + pack.ContractFrame = &protocol.Frame{ + MessageType: protocol.MessageType_TransferContract, MessageBytes: contractMessageBytes, } } @@ -1892,10 +1869,10 @@ func (self *SendSequence) receiveAck(messageId Id, selective bool) { for ; i < len(self.sendItems); i += 1 { implicitItem := self.sendItems[i] if item.sequenceNumber < implicitItem.sequenceNumber { - glog.V(2).Infof("[s]ack %d <> %d/%d (stop) %s->%s...%s s(%s)\n", item.sequenceNumber, implicitItem.sequenceNumber, self.nextSequenceNumber - 1, self.client.ClientTag(), self.intermediaryIds, self.destination.DestinationId, self.destination.StreamId) + glog.V(2).Infof("[s]ack %d <> %d/%d (stop) %s->%s...%s s(%s)\n", item.sequenceNumber, implicitItem.sequenceNumber, self.nextSequenceNumber-1, self.client.ClientTag(), self.intermediaryIds, self.destination.DestinationId, self.destination.StreamId) break } - + var a int var b ByteCount if glog.V(2) { @@ -1913,13 +1890,13 @@ func (self *SendSequence) receiveAck(messageId Id, selective bool) { if glog.V(2) { c, d := self.resendQueue.QueueSize() - glog.Infof("[s]ack %d <> %d/%d (pass %d->%d %dB->%dB) %s->%s...%s s(%s)\n", item.sequenceNumber, implicitItem.sequenceNumber, self.nextSequenceNumber - 1, a, c, b, d, self.client.ClientTag(), self.intermediaryIds, self.destination.DestinationId, self.destination.StreamId) + glog.Infof("[s]ack %d <> %d/%d (pass %d->%d %dB->%dB) %s->%s...%s s(%s)\n", item.sequenceNumber, implicitItem.sequenceNumber, self.nextSequenceNumber-1, a, c, b, d, self.client.ClientTag(), self.intermediaryIds, self.destination.DestinationId, self.destination.StreamId) } } self.sendItems = self.sendItems[i:] if glog.V(2) { a, b := self.resendQueue.QueueSize() - glog.Infof("[s]ack %d/%d (stop %d %dB %d) %s->%s...%s s(%s)\n", item.sequenceNumber, self.nextSequenceNumber - 1, a, b, len(self.sendItems), self.client.ClientTag(), self.intermediaryIds, self.destination.DestinationId, self.destination.StreamId) + glog.Infof("[s]ack %d/%d (stop %d %dB %d) %s->%s...%s s(%s)\n", item.sequenceNumber, self.nextSequenceNumber-1, a, b, len(self.sendItems), self.client.ClientTag(), self.intermediaryIds, self.destination.DestinationId, self.destination.StreamId) } } @@ -1988,7 +1965,7 @@ func (self *SendSequence) Close() { func() { for { select { - case sendPack, ok := <- self.packs: + case sendPack, ok := <-self.packs: if !ok { return } @@ -2007,26 +1984,25 @@ func (self *SendSequence) Cancel() { type sendItem struct { transferItem - contractId *Id - head bool - hasContractFrame bool - sendTime time.Time - resendTime time.Time - sendCount int + contractId *Id + head bool + hasContractFrame bool + sendTime time.Time + resendTime time.Time + sendCount int transferFrameBytes []byte - ackCallback AckFunction + ackCallback AckFunction // messageType protocol.MessageType } - // a send event queue which is the union of: // - resend times // - ack timeouts type resendQueue = transferQueue[*sendItem] func newResendQueue() *resendQueue { - return newTransferQueue[*sendItem](func(a *sendItem, b *sendItem)(int) { + return newTransferQueue[*sendItem](func(a *sendItem, b *sendItem) int { if a.resendTime.Before(b.resendTime) { return -1 } else if b.resendTime.Before(a.resendTime) { @@ -2037,9 +2013,8 @@ func newResendQueue() *resendQueue { }) } - type ReceiveBufferSettings struct { - GapTimeout time.Duration + GapTimeout time.Duration IdleTimeout time.Duration SequenceBufferSize int @@ -2061,15 +2036,13 @@ type ReceiveBufferSettings struct { ReceiveQueueMaxByteCount ByteCount } - type receiveSequenceId struct { - Source TransferPath + Source TransferPath SequenceId Id } - type ReceiveBuffer struct { - ctx context.Context + ctx context.Context client *Client receiveBufferSettings *ReceiveBufferSettings @@ -2080,23 +2053,23 @@ type ReceiveBuffer struct { } func NewReceiveBuffer(ctx context.Context, - client *Client, - receiveBufferSettings *ReceiveBufferSettings) *ReceiveBuffer { + client *Client, + receiveBufferSettings *ReceiveBufferSettings) *ReceiveBuffer { return &ReceiveBuffer{ - ctx: ctx, - client: client, + ctx: ctx, + client: client, receiveBufferSettings: receiveBufferSettings, - receiveSequences: map[receiveSequenceId]*ReceiveSequence{}, + receiveSequences: map[receiveSequenceId]*ReceiveSequence{}, } } func (self *ReceiveBuffer) Pack(receivePack *ReceivePack, timeout time.Duration) (bool, error) { receiveSequenceId := receiveSequenceId{ - Source: receivePack.Source, + Source: receivePack.Source, SequenceId: receivePack.SequenceId, } - initReceiveSequence := func(skip *ReceiveSequence)(*ReceiveSequence) { + initReceiveSequence := func(skip *ReceiveSequence) *ReceiveSequence { self.mutex.Lock() defer self.mutex.Unlock() @@ -2137,7 +2110,7 @@ func (self *ReceiveBuffer) Pack(receivePack *ReceivePack, timeout time.Duration) var err error for i := 0; i < 2; i += 1 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") default: } @@ -2155,7 +2128,7 @@ func (self *ReceiveBuffer) ReceiveQueueSize(source TransferPath, sequenceId Id) defer self.mutex.Unlock() receiveSequenceId := receiveSequenceId{ - Source: source, + Source: source, SequenceId: sequenceId, } if receiveSequence, ok := self.receiveSequences[receiveSequenceId]; ok { @@ -2197,14 +2170,13 @@ func (self *ReceiveBuffer) Flush() { } } - type ReceiveSequence struct { - ctx context.Context + ctx context.Context cancel context.CancelFunc client *Client - source TransferPath + source TransferPath sequenceId Id receiveBufferSettings *ReceiveBufferSettings @@ -2212,9 +2184,9 @@ type ReceiveSequence struct { receiveContract *sequenceContract packMutex sync.Mutex - packs chan *ReceivePack + packs chan *ReceivePack - receiveQueue *receiveQueue + receiveQueue *receiveQueue nextSequenceNumber uint64 idleCondition *IdleCondition @@ -2225,25 +2197,25 @@ type ReceiveSequence struct { } func NewReceiveSequence( - ctx context.Context, - client *Client, - source TransferPath, - sequenceId Id, - receiveBufferSettings *ReceiveBufferSettings) *ReceiveSequence { + ctx context.Context, + client *Client, + source TransferPath, + sequenceId Id, + receiveBufferSettings *ReceiveBufferSettings) *ReceiveSequence { cancelCtx, cancel := context.WithCancel(ctx) return &ReceiveSequence{ - ctx: cancelCtx, - cancel: cancel, - client: client, - source: source, - sequenceId: sequenceId, + ctx: cancelCtx, + cancel: cancel, + client: client, + source: source, + sequenceId: sequenceId, receiveBufferSettings: receiveBufferSettings, - receiveContract: nil, - packs: make(chan *ReceivePack, receiveBufferSettings.SequenceBufferSize), - receiveQueue: newReceiveQueue(), - nextSequenceNumber: 0, - idleCondition: NewIdleCondition(), - ackWindow: newSequenceAckWindow(), + receiveContract: nil, + packs: make(chan *ReceivePack, receiveBufferSettings.SequenceBufferSize), + receiveQueue: newReceiveQueue(), + nextSequenceNumber: 0, + idleCondition: NewIdleCondition(), + ackWindow: newSequenceAckWindow(), } } @@ -2257,7 +2229,7 @@ func (self *ReceiveSequence) Pack(receivePack *ReceivePack, timeout time.Duratio defer self.packMutex.Unlock() select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") default: } @@ -2269,14 +2241,14 @@ func (self *ReceiveSequence) Pack(receivePack *ReceivePack, timeout time.Duratio if timeout < 0 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") case self.packs <- receivePack: return true, nil } } else if timeout == 0 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") case self.packs <- receivePack: return true, nil @@ -2285,11 +2257,11 @@ func (self *ReceiveSequence) Pack(receivePack *ReceivePack, timeout time.Duratio } } else { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") case self.packs <- receivePack: return true, nil - case <- time.After(timeout): + case <-time.After(timeout): return false, nil } } @@ -2341,9 +2313,9 @@ func (self *ReceiveSequence) Run() { writeAck := func(sendAck *sequenceAck) { ack := &protocol.Ack{ - MessageId: sendAck.messageId.Bytes(), + MessageId: sendAck.messageId.Bytes(), SequenceId: self.sequenceId.Bytes(), - Selective: sendAck.selective, + Selective: sendAck.selective, } ackBytes, _ := proto.Marshal(ack) @@ -2352,14 +2324,14 @@ func (self *ReceiveSequence) Run() { transferFrame := &protocol.TransferFrame{ TransferPath: path.ToProtobuf(), Frame: &protocol.Frame{ - MessageType: protocol.MessageType_TransferAck, + MessageType: protocol.MessageType_TransferAck, MessageBytes: ackBytes, }, } transferFrameBytes, _ := proto.Marshal(transferFrame) - c := func()(error) { + c := func() error { return multiRouteWriter.Write( self.ctx, transferFrameBytes, @@ -2387,7 +2359,7 @@ func (self *ReceiveSequence) Run() { for { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return default: } @@ -2396,17 +2368,17 @@ func (self *ReceiveSequence) Run() { if ackSnapshot.ackUpdateCount == 0 && len(ackSnapshot.selectiveAcks) == 0 { // wait for one ack select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return - case <- ackSnapshot.ackNotify: + case <-ackSnapshot.ackNotify: } } if 0 < self.receiveBufferSettings.AckCompressTimeout { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return - case <- time.After(self.receiveBufferSettings.AckCompressTimeout): + case <-time.After(self.receiveBufferSettings.AckCompressTimeout): } } @@ -2416,9 +2388,9 @@ func (self *ReceiveSequence) Run() { } for messageId, sequenceNumber := range ackSnapshot.selectiveAcks { writeAck(&sequenceAck{ - messageId: messageId, + messageId: messageId, sequenceNumber: sequenceNumber, - selective: true, + selective: true, }) } } @@ -2427,7 +2399,7 @@ func (self *ReceiveSequence) Run() { for { receiveTime := time.Now() var timeout time.Duration - + if queueSize, _ := self.receiveQueue.QueueSize(); 0 == queueSize { timeout = self.receiveBufferSettings.IdleTimeout } else { @@ -2454,8 +2426,7 @@ func (self *ReceiveSequence) Run() { // item.sequenceNumber <= self.nextSequenceNumber self.receiveQueue.RemoveByMessageId(item.messageId) - - + if self.nextSequenceNumber == item.sequenceNumber { // this item is the head of sequence if err := self.registerContracts(item); err != nil { @@ -2463,7 +2434,7 @@ func (self *ReceiveSequence) Run() { return } if self.updateContract(item) { - glog.V(1).Infof("[r]seq+ %d->%d (queue) %s<-%s s(%s)\n", self.nextSequenceNumber, self.nextSequenceNumber + 1, self.client.ClientTag(), self.source.SourceId, self.source.StreamId) + glog.V(1).Infof("[r]seq+ %d->%d (queue) %s<-%s s(%s)\n", self.nextSequenceNumber, self.nextSequenceNumber+1, self.client.ClientTag(), self.source.SourceId, self.source.StreamId) self.nextSequenceNumber = self.nextSequenceNumber + 1 self.receiveHead(item) } else { @@ -2482,9 +2453,9 @@ func (self *ReceiveSequence) Run() { checkpointId := self.idleCondition.Checkpoint() select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return - case receivePack, ok := <- self.packs: + case receivePack, ok := <-self.packs: if !ok { return } @@ -2498,7 +2469,7 @@ func (self *ReceiveSequence) Run() { self.peerAudit.Update(func(a *PeerAudit) { a.badMessage(receivePack.MessageByteCount) }) - return + return } else if !received { glog.V(1).Infof("[r]drop nack %s<-%s s(%s)\n", self.client.ClientTag(), self.source.SourceId, self.source.StreamId) // drop the message @@ -2507,7 +2478,7 @@ func (self *ReceiveSequence) Run() { }) } - // note messages of `size < MinMessageByteCount` get counted as `MinMessageByteCount` against the contract + // note messages of `size < MinMessageByteCount` get counted as `MinMessageByteCount` against the contract } else { received, err := self.receive(receivePack) if err != nil { @@ -2517,7 +2488,7 @@ func (self *ReceiveSequence) Run() { self.peerAudit.Update(func(a *PeerAudit) { a.badMessage(receivePack.MessageByteCount) }) - return + return } else if !received { glog.V(1).Infof("[r]drop ack %s<-%s s(%s)\n", self.client.ClientTag(), self.source.SourceId, self.source.StreamId) // drop the message @@ -2526,7 +2497,7 @@ func (self *ReceiveSequence) Run() { }) } } - case <- time.After(timeout): + case <-time.After(timeout): if 0 == self.receiveQueue.Len() { // idle timeout if self.idleCondition.Close(checkpointId) { @@ -2539,22 +2510,22 @@ func (self *ReceiveSequence) Run() { // FIXME audit SendCount is currently not being updated /* - // check the resend abuse limits - // resends can appear normal but waste bandwidth - abuse := false - self.peerAudit.Update(func(a *PeerAudit) { - if self.receiveBufferSettings.ResendAbuseThreshold <= a.ResendCount { - resendByteCountAbuse := ByteCount(float64(a.SendByteCount) * self.receiveBufferSettings.ResendAbuseMultiple) <= a.ResendByteCount - resendCountAbuse := int(float64(a.SendCount) * self.receiveBufferSettings.ResendAbuseMultiple) <= a.ResendCount - abuse = resendByteCountAbuse || resendCountAbuse - a.Abuse = abuse + // check the resend abuse limits + // resends can appear normal but waste bandwidth + abuse := false + self.peerAudit.Update(func(a *PeerAudit) { + if self.receiveBufferSettings.ResendAbuseThreshold <= a.ResendCount { + resendByteCountAbuse := ByteCount(float64(a.SendByteCount) * self.receiveBufferSettings.ResendAbuseMultiple) <= a.ResendByteCount + resendCountAbuse := int(float64(a.SendCount) * self.receiveBufferSettings.ResendAbuseMultiple) <= a.ResendCount + abuse = resendByteCountAbuse || resendCountAbuse + a.Abuse = abuse + } + }) + if abuse { + // close the sequence + self.routeManager.DowngradeReceiverConnection(self.sourceId) + return } - }) - if abuse { - // close the sequence - self.routeManager.DowngradeReceiverConnection(self.sourceId) - return - } */ } } @@ -2562,8 +2533,8 @@ func (self *ReceiveSequence) Run() { func (self *ReceiveSequence) sendAck(sequenceNumber uint64, messageId Id, selective bool) { ack := &sequenceAck{ sequenceNumber: sequenceNumber, - messageId: messageId, - selective: selective, + messageId: messageId, + selective: selective, } self.ackWindow.Update(ack) } @@ -2583,21 +2554,20 @@ func (self *ReceiveSequence) receive(receivePack *ReceivePack) (bool, error) { item := &receiveItem{ transferItem: transferItem{ - messageId: messageId, - sequenceNumber: sequenceNumber, + messageId: messageId, + sequenceNumber: sequenceNumber, messageByteCount: receivePack.MessageByteCount, }, - - contractId: contractId, - receiveTime: receiveTime, - frames: receivePack.Pack.Frames, - contractFrame: receivePack.Pack.ContractFrame, + + contractId: contractId, + receiveTime: receiveTime, + frames: receivePack.Pack.Frames, + contractFrame: receivePack.Pack.ContractFrame, receiveCallback: receivePack.ReceiveCallback, - head: receivePack.Pack.Head, - ack: !receivePack.Pack.Nack, + head: receivePack.Pack.Head, + ack: !receivePack.Pack.Nack, } - // this case happens when the receiver is reformed or loses state. // the sequence id guarantees the sender is the same for the sequence // past head items are retransmits. Future head items depend on previous ack, @@ -2625,7 +2595,7 @@ func (self *ReceiveSequence) receive(receivePack *ReceivePack) (bool, error) { if sequenceNumber <= self.nextSequenceNumber { if self.nextSequenceNumber == sequenceNumber { // this item is the head of sequence - glog.V(2).Infof("[r]seq+ %d->%d %s<-%s s(%s)\n", self.nextSequenceNumber, self.nextSequenceNumber + 1, self.client.ClientTag(), self.source.SourceId, self.source.StreamId) + glog.V(2).Infof("[r]seq+ %d->%d %s<-%s s(%s)\n", self.nextSequenceNumber, self.nextSequenceNumber+1, self.client.ClientTag(), self.source.SourceId, self.source.StreamId) self.nextSequenceNumber = self.nextSequenceNumber + 1 if err := self.registerContracts(item); err != nil { @@ -2649,23 +2619,23 @@ func (self *ReceiveSequence) receive(receivePack *ReceivePack) (bool, error) { } } else { // store only up to a max size in the receive queue - canQueue := func(byteCount ByteCount)(bool) { - // always allow at least one item in the receive queue - queueSize, queueByteCount := self.receiveQueue.QueueSize() - if 0 == queueSize { - return true - } - return queueByteCount + byteCount < self.receiveBufferSettings.ReceiveQueueMaxByteCount + canQueue := func(byteCount ByteCount) bool { + // always allow at least one item in the receive queue + queueSize, queueByteCount := self.receiveQueue.QueueSize() + if 0 == queueSize { + return true + } + return queueByteCount+byteCount < self.receiveBufferSettings.ReceiveQueueMaxByteCount } // remove later items to fit for !canQueue(receivePack.MessageByteCount) { - lastItem := self.receiveQueue.PeekLast() - if receivePack.Pack.SequenceNumber < lastItem.sequenceNumber { - self.receiveQueue.RemoveByMessageId(lastItem.messageId) - } else { - break - } + lastItem := self.receiveQueue.PeekLast() + if receivePack.Pack.SequenceNumber < lastItem.sequenceNumber { + self.receiveQueue.RemoveByMessageId(lastItem.messageId) + } else { + break + } } if canQueue(receivePack.MessageByteCount) { @@ -2695,17 +2665,17 @@ func (self *ReceiveSequence) receiveNack(receivePack *ReceivePack) (bool, error) item := &receiveItem{ transferItem: transferItem{ - messageId: messageId, - sequenceNumber: sequenceNumber, + messageId: messageId, + sequenceNumber: sequenceNumber, messageByteCount: receivePack.MessageByteCount, }, - contractId: contractId, - receiveTime: receiveTime, - frames: receivePack.Pack.Frames, - contractFrame: receivePack.Pack.ContractFrame, + contractId: contractId, + receiveTime: receiveTime, + frames: receivePack.Pack.Frames, + contractFrame: receivePack.Pack.ContractFrame, receiveCallback: receivePack.ReceiveCallback, - head: receivePack.Pack.Head, - ack: !receivePack.Pack.Nack, + head: receivePack.Pack.Head, + ack: !receivePack.Pack.Nack, } if err := self.registerContracts(item); err != nil { @@ -2772,9 +2742,9 @@ func (self *ReceiveSequence) registerContracts(item *receiveItem) error { // check the hmac with the local provider secret key if !self.client.ContractManager().Verify( - contract.StoredContractHmac, - contract.StoredContractBytes, - contract.ProvideMode) { + contract.StoredContractHmac, + contract.StoredContractBytes, + contract.ProvideMode) { glog.Infof("[r]%s<-%s s(%s) exit contract verification failed (%s)\n", self.client.ClientTag(), self.source.SourceId, self.source.StreamId, contract.ProvideMode) // bad contract // close sequence @@ -2836,7 +2806,7 @@ func (self *ReceiveSequence) updateContract(item *receiveItem) bool { if self.receiveContract != nil && self.receiveContract.update(item.messageByteCount) { return true } - // `receiveNoContract` is a mutual configuration + // `receiveNoContract` is a mutual configuration // both sides must configure themselves to require no contract from each other if !self.source.IsStream() && self.client.ContractManager().ReceiveNoContract(self.source.SourceId) { return true @@ -2860,25 +2830,23 @@ func (self *ReceiveSequence) Cancel() { self.cancel() } - type receiveItem struct { transferItem - contractId *Id - head bool - receiveTime time.Time - frames []*protocol.Frame - contractFrame *protocol.Frame + contractId *Id + head bool + receiveTime time.Time + frames []*protocol.Frame + contractFrame *protocol.Frame receiveCallback ReceiveFunction - ack bool + ack bool } - // ordered by sequenceNumber type receiveQueue = transferQueue[*receiveItem] func newReceiveQueue() *receiveQueue { - return newTransferQueue[*receiveItem](func(a *receiveItem, b *receiveItem)(int) { + return newTransferQueue[*receiveItem](func(a *receiveItem, b *receiveItem) int { if a.sequenceNumber < b.sequenceNumber { return -1 } else if b.sequenceNumber < a.sequenceNumber { @@ -2889,34 +2857,33 @@ func newReceiveQueue() *receiveQueue { }) } - type sequenceAck struct { sequenceNumber uint64 - messageId Id - selective bool + messageId Id + selective bool } type sequenceAckWindowSnapshot struct { - ackNotify <-chan struct{} - headAck *sequenceAck + ackNotify <-chan struct{} + headAck *sequenceAck ackUpdateCount int - selectiveAcks map[Id]uint64 + selectiveAcks map[Id]uint64 } type sequenceAckWindow struct { - ackMonitor *Monitor - ackLock sync.Mutex - headAck *sequenceAck + ackMonitor *Monitor + ackLock sync.Mutex + headAck *sequenceAck ackUpdateCount int - selectiveAcks map[Id]uint64 + selectiveAcks map[Id]uint64 } func newSequenceAckWindow() *sequenceAckWindow { return &sequenceAckWindow{ - ackMonitor: NewMonitor(), - headAck: nil, + ackMonitor: NewMonitor(), + headAck: nil, ackUpdateCount: 0, - selectiveAcks: map[Id]uint64{}, + selectiveAcks: map[Id]uint64{}, } } @@ -2959,10 +2926,10 @@ func (self *sequenceAckWindow) Snapshot(reset bool) *sequenceAckWindowSnapshot { } snapshot := &sequenceAckWindowSnapshot{ - ackNotify: self.ackMonitor.NotifyChannel(), - headAck: self.headAck, + ackNotify: self.ackMonitor.NotifyChannel(), + headAck: self.headAck, ackUpdateCount: self.ackUpdateCount, - selectiveAcks: selectiveAcksAfterHead, + selectiveAcks: selectiveAcksAfterHead, } if reset { @@ -2974,20 +2941,19 @@ func (self *sequenceAckWindow) Snapshot(reset bool) *sequenceAckWindowSnapshot { return snapshot } - type sequenceContract struct { - tag string - contract *protocol.Contract - contractId Id - transferByteCount ByteCount + tag string + contract *protocol.Contract + contractId Id + transferByteCount ByteCount effectiveTransferByteCount ByteCount - provideMode protocol.ProvideMode + provideMode protocol.ProvideMode minUpdateByteCount ByteCount path TransferPath - - ackedByteCount ByteCount + + ackedByteCount ByteCount unackedByteCount ByteCount } @@ -3012,25 +2978,24 @@ func newSequenceContract(tag string, contract *protocol.Contract, minUpdateByteC return nil, err } - return &sequenceContract{ - tag: tag, - contract: contract, - contractId: contractId, - transferByteCount: ByteCount(storedContract.TransferByteCount), + tag: tag, + contract: contract, + contractId: contractId, + transferByteCount: ByteCount(storedContract.TransferByteCount), effectiveTransferByteCount: ByteCount(float32(storedContract.TransferByteCount) * contractFillFraction), - provideMode: contract.ProvideMode, - minUpdateByteCount: minUpdateByteCount, - path: path, - ackedByteCount: ByteCount(0), - unackedByteCount: ByteCount(0), + provideMode: contract.ProvideMode, + minUpdateByteCount: minUpdateByteCount, + path: path, + ackedByteCount: ByteCount(0), + unackedByteCount: ByteCount(0), }, nil } func (self *sequenceContract) update(byteCount ByteCount) bool { effectiveByteCount := max(self.minUpdateByteCount, byteCount) - if self.effectiveTransferByteCount < self.ackedByteCount + self.unackedByteCount + effectiveByteCount { + if self.effectiveTransferByteCount < self.ackedByteCount+self.unackedByteCount+effectiveByteCount { // doesn't fit in contract if glog.V(1) { glog.Infof( @@ -3038,10 +3003,10 @@ func (self *sequenceContract) update(byteCount ByteCount) bool { self.tag, self.contractId, effectiveByteCount, - self.ackedByteCount + self.unackedByteCount + effectiveByteCount, - self.ackedByteCount + self.unackedByteCount, + self.ackedByteCount+self.unackedByteCount+effectiveByteCount, + self.ackedByteCount+self.unackedByteCount, self.effectiveTransferByteCount, - 100.0 * float32(self.ackedByteCount + self.unackedByteCount) / float32(self.effectiveTransferByteCount), + 100.0*float32(self.ackedByteCount+self.unackedByteCount)/float32(self.effectiveTransferByteCount), ) } return false @@ -3053,10 +3018,10 @@ func (self *sequenceContract) update(byteCount ByteCount) bool { self.tag, self.contractId, effectiveByteCount, - self.ackedByteCount + self.unackedByteCount, - self.ackedByteCount + self.unackedByteCount, + self.ackedByteCount+self.unackedByteCount, + self.ackedByteCount+self.unackedByteCount, self.effectiveTransferByteCount, - 100.0 * float32(self.ackedByteCount + self.unackedByteCount) / float32(self.effectiveTransferByteCount), + 100.0*float32(self.ackedByteCount+self.unackedByteCount)/float32(self.effectiveTransferByteCount), ) } return true @@ -3073,7 +3038,6 @@ func (self *sequenceContract) ack(byteCount ByteCount) { self.ackedByteCount += effectiveByteCount } - type ForwardBufferSettings struct { IdleTimeout time.Duration @@ -3082,9 +3046,8 @@ type ForwardBufferSettings struct { WriteTimeout time.Duration } - type ForwardBuffer struct { - ctx context.Context + ctx context.Context client *Client forwardBufferSettings *ForwardBufferSettings @@ -3095,18 +3058,18 @@ type ForwardBuffer struct { } func NewForwardBuffer(ctx context.Context, - client *Client, - forwardBufferSettings *ForwardBufferSettings) *ForwardBuffer { + client *Client, + forwardBufferSettings *ForwardBufferSettings) *ForwardBuffer { return &ForwardBuffer{ - ctx: ctx, - client: client, + ctx: ctx, + client: client, forwardBufferSettings: forwardBufferSettings, - forwardSequences: map[TransferPath]*ForwardSequence{}, + forwardSequences: map[TransferPath]*ForwardSequence{}, } } func (self *ForwardBuffer) Pack(forwardPack *ForwardPack, timeout time.Duration) (bool, error) { - initForwardSequence := func(skip *ForwardSequence)(*ForwardSequence) { + initForwardSequence := func(skip *ForwardSequence) *ForwardSequence { self.mutex.Lock() defer self.mutex.Unlock() @@ -3145,7 +3108,7 @@ func (self *ForwardBuffer) Pack(forwardPack *ForwardPack, timeout time.Duration) var err error for i := 0; i < 2; i += 1 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") default: } @@ -3191,13 +3154,12 @@ func (self *ForwardBuffer) Flush() { } } - type ForwardSequence struct { - ctx context.Context + ctx context.Context cancel context.CancelFunc - client *Client - clientId Id + client *Client + clientId Id clientTag string destination TransferPath @@ -3205,29 +3167,27 @@ type ForwardSequence struct { forwardBufferSettings *ForwardBufferSettings packMutex sync.Mutex - packs chan *ForwardPack + packs chan *ForwardPack idleCondition *IdleCondition multiRouteWriter MultiRouteWriter - - } func NewForwardSequence( - ctx context.Context, - client *Client, - destination TransferPath, - forwardBufferSettings *ForwardBufferSettings) *ForwardSequence { + ctx context.Context, + client *Client, + destination TransferPath, + forwardBufferSettings *ForwardBufferSettings) *ForwardSequence { cancelCtx, cancel := context.WithCancel(ctx) return &ForwardSequence{ - ctx: cancelCtx, - cancel: cancel, - client: client, - destination: destination, + ctx: cancelCtx, + cancel: cancel, + client: client, + destination: destination, forwardBufferSettings: forwardBufferSettings, - packs: make(chan *ForwardPack, forwardBufferSettings.SequenceBufferSize), - idleCondition: NewIdleCondition(), + packs: make(chan *ForwardPack, forwardBufferSettings.SequenceBufferSize), + idleCondition: NewIdleCondition(), } } @@ -3235,13 +3195,13 @@ func NewForwardSequence( func (self *ForwardSequence) Pack(forwardPack *ForwardPack, timeout time.Duration) (bool, error) { self.packMutex.Lock() defer self.packMutex.Unlock() - + select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") default: } - + if !self.idleCondition.UpdateOpen() { return false, errors.New("Done.") } @@ -3249,14 +3209,14 @@ func (self *ForwardSequence) Pack(forwardPack *ForwardPack, timeout time.Duratio if timeout < 0 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") case self.packs <- forwardPack: return true, nil } } else if timeout == 0 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") case self.packs <- forwardPack: return true, nil @@ -3265,14 +3225,14 @@ func (self *ForwardSequence) Pack(forwardPack *ForwardPack, timeout time.Duratio } } else { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") case self.packs <- forwardPack: return true, nil - case <- time.After(timeout): + case <-time.After(timeout): return false, nil } - } + } } func (self *ForwardSequence) Run() { @@ -3284,13 +3244,13 @@ func (self *ForwardSequence) Run() { for { checkpointId := self.idleCondition.Checkpoint() select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return - case forwardPack, ok := <- self.packs: + case forwardPack, ok := <-self.packs: if !ok { return } - c := func()(error) { + c := func() error { return self.multiRouteWriter.Write(self.ctx, forwardPack.TransferFrameBytes, self.forwardBufferSettings.WriteTimeout) } if glog.V(2) { @@ -3304,7 +3264,7 @@ func (self *ForwardSequence) Run() { glog.Infof("[f]drop = %s", err) } } - case <- time.After(self.forwardBufferSettings.IdleTimeout): + case <-time.After(self.forwardBufferSettings.IdleTimeout): if self.idleCondition.Close(checkpointId) { // close the sequence return @@ -3329,35 +3289,34 @@ func (self *ForwardSequence) Cancel() { self.cancel() } - type PeerAudit struct { - startTime time.Time - lastModifiedTime time.Time - Abuse bool - BadContractCount int - DiscardedByteCount ByteCount - DiscardedCount int - BadMessageByteCount ByteCount - BadMessageCount int - SendByteCount ByteCount - SendCount int - ResendByteCount ByteCount - ResendCount int + startTime time.Time + lastModifiedTime time.Time + Abuse bool + BadContractCount int + DiscardedByteCount ByteCount + DiscardedCount int + BadMessageByteCount ByteCount + BadMessageCount int + SendByteCount ByteCount + SendCount int + ResendByteCount ByteCount + ResendCount int } func NewPeerAudit(startTime time.Time) *PeerAudit { return &PeerAudit{ - startTime: startTime, - lastModifiedTime: startTime, - BadContractCount: 0, - DiscardedByteCount: ByteCount(0), - DiscardedCount: 0, - BadMessageByteCount: ByteCount(0), - BadMessageCount: 0, - SendByteCount: ByteCount(0), - SendCount: 0, - ResendByteCount: ByteCount(0), - ResendCount: 0, + startTime: startTime, + lastModifiedTime: startTime, + BadContractCount: 0, + DiscardedByteCount: ByteCount(0), + DiscardedCount: 0, + BadMessageByteCount: ByteCount(0), + BadMessageCount: 0, + SendByteCount: ByteCount(0), + SendCount: 0, + ResendByteCount: ByteCount(0), + ResendCount: 0, } } @@ -3385,10 +3344,9 @@ func (self *PeerAudit) resend(byteCount ByteCount) { self.ResendByteCount += byteCount } - type SequencePeerAudit struct { - client *Client - source TransferPath + client *Client + source TransferPath maxAuditDuration time.Duration peerAudit *PeerAudit @@ -3396,10 +3354,10 @@ type SequencePeerAudit struct { func NewSequencePeerAudit(client *Client, source TransferPath, maxAuditDuration time.Duration) *SequencePeerAudit { return &SequencePeerAudit{ - client: client, - source: source, + client: client, + source: source, maxAuditDuration: maxAuditDuration, - peerAudit: nil, + peerAudit: nil, } } @@ -3424,28 +3382,27 @@ func (self *SequencePeerAudit) Complete() { } peerAudit := &protocol.PeerAudit{ - PeerId: self.source.SourceId.Bytes(), - StreamId: self.source.StreamId.Bytes(), - Duration: uint64(math.Ceil((self.peerAudit.lastModifiedTime.Sub(self.peerAudit.startTime)).Seconds())), - Abuse: self.peerAudit.Abuse, - BadContractCount: uint64(self.peerAudit.BadContractCount), - DiscardedByteCount: uint64(self.peerAudit.DiscardedByteCount), - DiscardedCount: uint64(self.peerAudit.DiscardedCount), - BadMessageByteCount: uint64(self.peerAudit.BadMessageByteCount), - BadMessageCount: uint64(self.peerAudit.BadMessageCount), - SendByteCount: uint64(self.peerAudit.SendByteCount), - SendCount: uint64(self.peerAudit.SendCount), - ResendByteCount: uint64(self.peerAudit.ResendByteCount), - ResendCount: uint64(self.peerAudit.ResendCount), + PeerId: self.source.SourceId.Bytes(), + StreamId: self.source.StreamId.Bytes(), + Duration: uint64(math.Ceil((self.peerAudit.lastModifiedTime.Sub(self.peerAudit.startTime)).Seconds())), + Abuse: self.peerAudit.Abuse, + BadContractCount: uint64(self.peerAudit.BadContractCount), + DiscardedByteCount: uint64(self.peerAudit.DiscardedByteCount), + DiscardedCount: uint64(self.peerAudit.DiscardedCount), + BadMessageByteCount: uint64(self.peerAudit.BadMessageByteCount), + BadMessageCount: uint64(self.peerAudit.BadMessageCount), + SendByteCount: uint64(self.peerAudit.SendByteCount), + SendCount: uint64(self.peerAudit.SendCount), + ResendByteCount: uint64(self.peerAudit.ResendByteCount), + ResendCount: uint64(self.peerAudit.ResendCount), } self.client.ClientOob().SendControl( []*protocol.Frame{RequireToFrame(peerAudit)}, - func(resultFrames []*protocol.Frame, err error){}, + func(resultFrames []*protocol.Frame, err error) {}, ) self.peerAudit = nil } - // contract frames are not counted towards the message byte count // this is required since contracts can be attached post-hoc func MessageByteCount(frames []*protocol.Frame) ByteCount { diff --git a/connect/transfer_contract_manager.go b/connect/transfer_contract_manager.go index f3ee817..598b1ad 100644 --- a/connect/transfer_contract_manager.go +++ b/connect/transfer_contract_manager.go @@ -2,13 +2,15 @@ package connect import ( "context" - "time" "sync" + "time" + // "errors" "crypto/hmac" - "crypto/sha256" "crypto/rand" + "crypto/sha256" "fmt" + // "slices" // "runtime/debug" @@ -18,18 +20,16 @@ import ( "github.com/golang/glog" - "bringyour.com/protocol" + "github.com/bringyour/connect/protocol" ) - // manage contracts which are embedded into each transfer sequence - type ContractKey struct { - Destination TransferPath - IntermediaryIds MultiHopId + Destination TransferPath + IntermediaryIds MultiHopId CompanionContract bool - ForceStream bool + ForceStream bool } func (self ContractKey) Legacy() ContractKey { @@ -38,28 +38,26 @@ func (self ContractKey) Legacy() ContractKey { } } - type ContractErrorFunction = func(contractError protocol.ContractError) - type ContractManagerStats struct { - ContractOpenCount int64 + ContractOpenCount int64 ContractCloseCount int64 // contract id -> byte count ContractOpenByteCounts map[Id]ByteCount // contract id -> contract key - ContractOpenKeys map[Id]ContractKey - ContractCloseByteCount ByteCount + ContractOpenKeys map[Id]ContractKey + ContractCloseByteCount ByteCount ReceiveContractCloseByteCount ByteCount } -func NewContractManagerStats() *ContractManagerStats{ +func NewContractManagerStats() *ContractManagerStats { return &ContractManagerStats{ - ContractOpenCount: 0, - ContractCloseCount: 0, - ContractOpenByteCounts: map[Id]ByteCount{}, - ContractOpenKeys: map[Id]ContractKey{}, - ContractCloseByteCount: 0, + ContractOpenCount: 0, + ContractCloseCount: 0, + ContractOpenByteCounts: map[Id]ByteCount{}, + ContractOpenKeys: map[Id]ContractKey{}, + ContractCloseByteCount: 0, ReceiveContractCloseByteCount: 0, } } @@ -72,7 +70,6 @@ func (self *ContractManagerStats) ContractOpenByteCount() ByteCount { return netContractOpenByteCount } - func DefaultContractManagerSettings() *ContractManagerSettings { // NETWORK EVENT: at the enable contracts date, all clients will require contracts // up to that time, contracts are optional for the sender and match for the receiver @@ -96,7 +93,6 @@ func DefaultContractManagerSettingsNoNetworkEvents() *ContractManagerSettings { return settings } - type ContractManagerSettings struct { StandardContractTransferByteCount ByteCount @@ -111,9 +107,8 @@ func (self *ContractManagerSettings) ContractsEnabled() bool { return self.NetworkEventTimeEnableContracts.Before(time.Now()) } - type ContractManager struct { - ctx context.Context + ctx context.Context client *Client settings *ContractManagerSettings @@ -123,9 +118,9 @@ type ContractManager struct { provideSecretKeys map[protocol.ProvideMode][]byte destinationContracts map[ContractKey]*contractQueue - + receiveNoContractClientIds map[Id]bool - sendNoContractClientIds map[Id]bool + sendNoContractClientIds map[Id]bool contractErrorCallbacks *CallbackList[ContractErrorFunction] @@ -141,29 +136,29 @@ func NewContractManager( client *Client, settings *ContractManagerSettings, ) *ContractManager { - // at a minimum + // at a minimum // - messages to/from the platform (ControlId) do not need a contract // this is because the platform is needed to create contracts // - messages to self do not need a contract receiveNoContractClientIds := map[Id]bool{ - ControlId: true, + ControlId: true, client.ClientId(): true, } sendNoContractClientIds := map[Id]bool{ - ControlId: true, + ControlId: true, client.ClientId(): true, } contractManager := &ContractManager{ - ctx: ctx, - client: client, - settings: settings, - provideSecretKeys: map[protocol.ProvideMode][]byte{}, - destinationContracts: map[ContractKey]*contractQueue{}, + ctx: ctx, + client: client, + settings: settings, + provideSecretKeys: map[protocol.ProvideMode][]byte{}, + destinationContracts: map[ContractKey]*contractQueue{}, receiveNoContractClientIds: receiveNoContractClientIds, - sendNoContractClientIds: sendNoContractClientIds, - contractErrorCallbacks: NewCallbackList[ContractErrorFunction](), - localStats: NewContractManagerStats(), + sendNoContractClientIds: sendNoContractClientIds, + contractErrorCallbacks: NewCallbackList[ContractErrorFunction](), + localStats: NewContractManagerStats(), } return contractManager @@ -191,7 +186,7 @@ func (self *ContractManager) Receive(source TransferPath, frames []*protocol.Fra contractErrors = append(contractErrors, frameContractErrors...) } for contract, contractKey := range contracts { - c := func()(error) { + c := func() error { return self.addContract(contractKey, contract) } if glog.V(2) { @@ -231,7 +226,7 @@ func (self *ContractManager) parseControlFrame(frame *protocol.Frame) ( case *protocol.CreateContractResult: if contractError := v.Error; contractError != nil { contractErrors = append(contractErrors, *contractError) - } else if contract := v.Contract; contract != nil { + } else if contract := v.Contract; contract != nil { contractKey := ContractKey{} var storedContract protocol.StoredContract @@ -279,7 +274,7 @@ func (self *ContractManager) contractError(contractError protocol.ContractError) } func (self *ContractManager) SetProvideModesWithReturnTraffic(provideModes map[protocol.ProvideMode]bool) { - self.SetProvideModesWithReturnTrafficWithAckCallback(provideModes, func(err error){}) + self.SetProvideModesWithReturnTrafficWithAckCallback(provideModes, func(err error) {}) } // clients must enable `ProvideMode_Stream` to allow return traffic @@ -291,7 +286,7 @@ func (self *ContractManager) SetProvideModesWithReturnTrafficWithAckCallback(pro } func (self *ContractManager) SetProvideModes(provideModes map[protocol.ProvideMode]bool) { - self.SetProvideModesWithAckCallback(provideModes, func(err error){}) + self.SetProvideModesWithAckCallback(provideModes, func(err error) {}) } func (self *ContractManager) SetProvideModesWithAckCallback(provideModes map[protocol.ProvideMode]bool, ackCallback func(err error)) { @@ -313,14 +308,14 @@ func (self *ContractManager) SetProvideModesWithAckCallback(provideModes map[pro if !ok { // generate a new key provideSecretKey = make([]byte, 32) - _, err := rand.Read(provideSecretKey) - if err != nil { - panic(err) - } + _, err := rand.Read(provideSecretKey) + if err != nil { + panic(err) + } self.provideSecretKeys[provideMode] = provideSecretKey } provideKeys = append(provideKeys, &protocol.ProvideKey{ - Mode: provideMode, + Mode: provideMode, ProvideSecretKey: provideSecretKey, }) } @@ -433,11 +428,11 @@ func (self *ContractManager) TakeContract( if timeout < 0 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return nil - case <- ctx.Done(): + case <-ctx.Done(): return nil - case <- notify: + case <-notify: } } else if timeout == 0 { return nil @@ -447,16 +442,16 @@ func (self *ContractManager) TakeContract( return nil } select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return nil - case <- ctx.Done(): + case <-ctx.Done(): return nil - case <- notify: - case <- time.After(remainingTimeout): + case <-notify: + case <-time.After(remainingTimeout): return nil } } - } + } } func (self *ContractManager) addContract(contractKey ContractKey, contract *protocol.Contract) error { @@ -509,13 +504,13 @@ func (self *ContractManager) CreateContract(contractKey ContractKey, timeout tim defer self.closeContractQueue(contractKey) createContract := &protocol.CreateContract{ - DestinationId: contractKey.Destination.DestinationId.Bytes(), - IntermediaryIds: contractKey.IntermediaryIds.Bytes(), - StreamId: contractKey.Destination.StreamId.Bytes(), + DestinationId: contractKey.Destination.DestinationId.Bytes(), + IntermediaryIds: contractKey.IntermediaryIds.Bytes(), + StreamId: contractKey.Destination.StreamId.Bytes(), TransferByteCount: uint64(self.settings.StandardContractTransferByteCount), - Companion: contractKey.CompanionContract, - ForceStream: &contractKey.ForceStream, - UsedContractIds: contractQueue.UsedContractIdBytes(), + Companion: contractKey.CompanionContract, + ForceStream: &contractKey.ForceStream, + UsedContractIds: contractQueue.UsedContractIdBytes(), } self.client.ClientOob().SendControl( []*protocol.Frame{RequireToFrame(createContract)}, @@ -572,10 +567,10 @@ func (self *ContractManager) CloseContractWithCheckpoint( }() closeContract := &protocol.CloseContract{ - ContractId: contractId.Bytes(), - AckedByteCount: uint64(ackedByteCount), + ContractId: contractId.Bytes(), + AckedByteCount: uint64(ackedByteCount), UnackedByteCount: uint64(unackedByteCount), - Checkpoint: checkpoint, + Checkpoint: checkpoint, } self.client.ClientOob().SendControl( []*protocol.Frame{RequireToFrame(closeContract)}, @@ -597,11 +592,11 @@ func (self *ContractManager) LocalStats() *ContractManagerStats { defer self.mutex.Unlock() return &ContractManagerStats{ - ContractOpenCount: self.localStats.ContractOpenCount, - ContractCloseCount: self.localStats.ContractCloseCount, + ContractOpenCount: self.localStats.ContractOpenCount, + ContractCloseCount: self.localStats.ContractCloseCount, ContractOpenByteCounts: maps.Clone(self.localStats.ContractOpenByteCounts), // ContractOpenDestinationIds: maps.Clone(self.localStats.ContractOpenDestinationIds), - ContractCloseByteCount: self.localStats.ContractCloseByteCount, + ContractCloseByteCount: self.localStats.ContractCloseByteCount, ReceiveContractCloseByteCount: self.localStats.ReceiveContractCloseByteCount, } } @@ -615,7 +610,7 @@ func (self *ContractManager) ResetLocalStats() { func (self *ContractManager) Flush(resetUsedContractIds bool) []Id { // close queued contracts - contracts := func()([]*protocol.Contract) { + contracts := func() []*protocol.Contract { self.mutex.Lock() defer self.mutex.Unlock() @@ -694,11 +689,10 @@ func (self *ContractManager) closeContractQueueWithForceRemove(contractKey Contr } } - type contractQueue struct { updateMonitor *Monitor - mutex sync.Mutex + mutex sync.Mutex openCount int contracts map[Id]*protocol.Contract // remember all added contract ids @@ -707,9 +701,9 @@ type contractQueue struct { func newContractQueue() *contractQueue { return &contractQueue{ - updateMonitor: NewMonitor(), - openCount: 0, - contracts: map[Id]*protocol.Contract{}, + updateMonitor: NewMonitor(), + openCount: 0, + contracts: map[Id]*protocol.Contract{}, usedContractIds: map[Id]bool{}, } } @@ -811,4 +805,3 @@ func (self *contractQueue) UsedContractIdBytes() [][]byte { } return usedContractIdBytes } - diff --git a/connect/transfer_contract_manager_test.go b/connect/transfer_contract_manager_test.go index a65634c..0be48cd 100644 --- a/connect/transfer_contract_manager_test.go +++ b/connect/transfer_contract_manager_test.go @@ -2,20 +2,20 @@ package connect import ( "context" - "testing" - "time" - // mathrand "math/rand" - "crypto/hmac" + "testing" + "time" + + // mathrand "math/rand" + "crypto/hmac" "crypto/sha256" - "github.com/go-playground/assert/v2" + "github.com/go-playground/assert/v2" - "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/proto" - "bringyour.com/protocol" + "github.com/bringyour/connect/protocol" ) - func TestTakeContract(t *testing.T) { // in parallel, add contracts, take contracts, and optionally return contract // make sure all created contracts get eventually taken @@ -25,37 +25,35 @@ func TestTakeContract(t *testing.T) { // contractReturnP := float32(0.5) timeout := 30 * time.Second - ctx := context.Background() clientId := NewId() client := NewClientWithDefaults(ctx, clientId, NewNoContractClientOob()) defer client.Cancel() contractManager := client.ContractManager() - + destinationId := NewId() contractManager.SetProvideModesWithReturnTraffic(map[protocol.ProvideMode]bool{ protocol.ProvideMode_Network: true, - protocol.ProvideMode_Public: true, + protocol.ProvideMode_Public: true, }) contracts := make(chan *protocol.Contract) go func() { - for i := 0; i < k * n; i += 1 { + for i := 0; i < k*n; i += 1 { contractId := NewId() contractByteCount := gib(1) - relationship := protocol.ProvideMode_Public provideSecretKey, ok := contractManager.GetProvideSecretKey(relationship) assert.Equal(t, true, ok) storedContract := &protocol.StoredContract{ - ContractId: contractId.Bytes(), + ContractId: contractId.Bytes(), TransferByteCount: uint64(contractByteCount), - SourceId: clientId.Bytes(), - DestinationId: destinationId.Bytes(), + SourceId: clientId.Bytes(), + DestinationId: destinationId.Bytes(), } storedContractBytes, err := proto.Marshal(storedContract) assert.Equal(t, nil, err) @@ -68,14 +66,13 @@ func TestTakeContract(t *testing.T) { result := &protocol.CreateContractResult{ Contract: &protocol.Contract{ StoredContractBytes: storedContractBytes, - StoredContractHmac: storedContractHmac, - ProvideMode: relationship, + StoredContractHmac: storedContractHmac, + ProvideMode: relationship, }, } frame, err := ToFrame(result) assert.Equal(t, nil, err) - contractManager.Receive(SourceId(ControlId), []*protocol.Frame{frame}, protocol.ProvideMode_Network) } }() @@ -92,12 +89,12 @@ func TestTakeContract(t *testing.T) { // // put back // contractManager.ReturnContract(ctx, destinationId, contract) // } else { - select { - case contracts <- contract: - case <- time.After(timeout): - t.FailNow() - } - i += 1 + select { + case contracts <- contract: + case <-time.After(timeout): + t.FailNow() + } + i += 1 // } } @@ -106,12 +103,11 @@ func TestTakeContract(t *testing.T) { }() } - contractIds := map[Id]bool{} - for i := 0; i < k * n; i += 1 { + for i := 0; i < k*n; i += 1 { select { - case contract := <- contracts: + case contract := <-contracts: var storedContract protocol.StoredContract err := proto.Unmarshal(contract.StoredContractBytes, &storedContract) assert.Equal(t, nil, err) @@ -122,12 +118,12 @@ func TestTakeContract(t *testing.T) { assert.Equal(t, false, contractIds[contractId]) contractIds[contractId] = true - case <- time.After(timeout): + case <-time.After(timeout): t.FailNow() } } - assert.Equal(t, k * n, len(contractIds)) + assert.Equal(t, k*n, len(contractIds)) // no more contractKey := ContractKey{ @@ -138,4 +134,3 @@ func TestTakeContract(t *testing.T) { // all the contracts are accounted for } - diff --git a/connect/transfer_oob_control.go b/connect/transfer_oob_control.go index 27a6fdf..920d075 100644 --- a/connect/transfer_oob_control.go +++ b/connect/transfer_oob_control.go @@ -8,22 +8,20 @@ import ( "google.golang.org/protobuf/proto" - "bringyour.com/protocol" + "github.com/bringyour/connect/protocol" ) - // control messages for a client out of band with the client sequence // some control messages require blocking response, but there is a potential deadlock // when a send blocks to wait for a control receive, or vice versa, since // all clients messages are multiplexed in the same client sequence // and the receive/send may be blocked on the send/receive -// for example think of a remote provider setup forwarding traffic as fast as possible +// for example think of a remote provider setup forwarding traffic as fast as possible // to an "echo" server with a finite buffer type OutOfBandControl interface { - SendControl(frames []*protocol.Frame, callback func(resultFrames []*protocol.Frame, err error)) + SendControl(frames []*protocol.Frame, callback func(resultFrames []*protocol.Frame, err error)) } - type ApiOutOfBandControl struct { api *BringYourApi } @@ -89,7 +87,6 @@ func (self *ApiOutOfBandControl) SendControl( ) } - type noContractClientOob struct { } diff --git a/connect/transfer_stream_manager.go b/connect/transfer_stream_manager.go index 744526b..4e1c3d1 100644 --- a/connect/transfer_stream_manager.go +++ b/connect/transfer_stream_manager.go @@ -2,56 +2,44 @@ package connect import ( "context" - "sync" "errors" "fmt" + "sync" "time" "github.com/golang/glog" - "bringyour.com/protocol" + "github.com/bringyour/connect/protocol" ) - - - func DefaultStreamManagerSettings() *StreamManagerSettings { return &StreamManagerSettings{ StreamBufferSettings: DefaultStreamBufferSettings(), - WebRtcSettings: DefaultWebRtcSettings(), + WebRtcSettings: DefaultWebRtcSettings(), } } - func DefaultStreamBufferSettings() *StreamBufferSettings { return &StreamBufferSettings{ - ReadTimeout: time.Duration(-1), - WriteTimeout: time.Duration(-1), + ReadTimeout: time.Duration(-1), + WriteTimeout: time.Duration(-1), P2pTransportSettings: DefaultP2pTransportSettings(), } } - - - type StreamManagerSettings struct { - StreamBufferSettings *StreamBufferSettings WebRtcSettings *WebRtcSettings - } - - - type StreamManager struct { ctx context.Context - + client *Client webRtcManager *WebRtcManager - + streamBuffer *StreamBuffer streamManagerSettings *StreamManagerSettings @@ -59,8 +47,8 @@ type StreamManager struct { func NewStreamManager(ctx context.Context, client *Client, streamManagerSettings *StreamManagerSettings) *StreamManager { streamManager := &StreamManager{ - ctx: ctx, - client: client, + ctx: ctx, + client: client, streamManagerSettings: streamManagerSettings, } @@ -115,7 +103,7 @@ func (self *StreamManager) handleControlFrame(frame *protocol.Frame) error { } destinationId = &destinationId_ } - + streamId, err := IdFromBytes(v.StreamId) if err != nil { return err @@ -139,22 +127,19 @@ func (self *StreamManager) IsStreamOpen(streamId Id) bool { return self.streamBuffer.IsStreamOpen(streamId) } - - type StreamBufferSettings struct { - ReadTimeout time.Duration + ReadTimeout time.Duration WriteTimeout time.Duration P2pTransportSettings *P2pTransportSettings } - type streamSequenceId struct { - SourceId Id - HasSource bool - DestinationId Id + SourceId Id + HasSource bool + DestinationId Id HasDestination bool - StreamId Id + StreamId Id } func newStreamSequenceId(sourceId *Id, destinationId *Id, streamId Id) streamSequenceId { @@ -172,7 +157,6 @@ func newStreamSequenceId(sourceId *Id, destinationId *Id, streamId Id) streamSeq return streamSequenceId } - type StreamBuffer struct { ctx context.Context @@ -180,17 +164,17 @@ type StreamBuffer struct { streamBufferSettings *StreamBufferSettings - mutex sync.Mutex - streamSequences map[streamSequenceId]*StreamSequence + mutex sync.Mutex + streamSequences map[streamSequenceId]*StreamSequence streamSequencesByStreamId map[Id]*StreamSequence } func NewStreamBuffer(ctx context.Context, streamManager *StreamManager, streamBufferSettings *StreamBufferSettings) *StreamBuffer { return &StreamBuffer{ - ctx: ctx, - streamManager: streamManager, - streamBufferSettings: streamBufferSettings, - streamSequences: map[streamSequenceId]*StreamSequence{}, + ctx: ctx, + streamManager: streamManager, + streamBufferSettings: streamBufferSettings, + streamSequences: map[streamSequenceId]*StreamSequence{}, streamSequencesByStreamId: map[Id]*StreamSequence{}, } } @@ -243,7 +227,7 @@ func (self *StreamBuffer) OpenStream(sourceId *Id, destinationId *Id, streamId I var err error for i := 0; i < 2; i += 1 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") default: } @@ -253,7 +237,7 @@ func (self *StreamBuffer) OpenStream(sourceId *Id, destinationId *Id, streamId I } // sequence closed } - return success, err + return success, err } func (self *StreamBuffer) CloseStream(streamId Id) { @@ -273,47 +257,45 @@ func (self *StreamBuffer) IsStreamOpen(streamId Id) bool { return ok } - type StreamSequence struct { - ctx context.Context + ctx context.Context cancel context.CancelFunc streamManager *StreamManager streamBufferSettings *StreamBufferSettings - sourceId *Id + sourceId *Id destinationId *Id - streamId Id + streamId Id idleCondition *IdleCondition } func NewStreamSequence( - ctx context.Context, - streamManager *StreamManager, - sourceId *Id, - destinationId *Id, - streamId Id, - streamBufferSettings *StreamBufferSettings) *StreamSequence { + ctx context.Context, + streamManager *StreamManager, + sourceId *Id, + destinationId *Id, + streamId Id, + streamBufferSettings *StreamBufferSettings) *StreamSequence { cancelCtx, cancel := context.WithCancel(ctx) return &StreamSequence{ - ctx: cancelCtx, - cancel: cancel, - streamManager: streamManager, + ctx: cancelCtx, + cancel: cancel, + streamManager: streamManager, streamBufferSettings: streamBufferSettings, - sourceId: sourceId, - destinationId: destinationId, - streamId: streamId, - idleCondition: NewIdleCondition(), + sourceId: sourceId, + destinationId: destinationId, + streamId: streamId, + idleCondition: NewIdleCondition(), } } - func (self *StreamSequence) Open() (bool, error) { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return false, errors.New("Done.") default: } @@ -326,7 +308,7 @@ func (self *StreamSequence) Open() (bool, error) { return true, nil } -func (self *StreamSequence) Run() { +func (self *StreamSequence) Run() { defer self.cancel() if self.sourceId == nil || self.destinationId == nil { @@ -420,7 +402,7 @@ func (self *StreamSequence) Run() { for { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return } @@ -430,7 +412,7 @@ func (self *StreamSequence) Run() { // idle timeout if self.idleCondition.Close(checkpointId) { // close the sequence - return + return } // else the sequence was opened again continue @@ -450,7 +432,7 @@ func (self *StreamSequence) Run() { } select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return } } @@ -462,4 +444,3 @@ func (self *StreamSequence) Cancel() { func (self *StreamSequence) Close() { self.cancel() } - diff --git a/connect/transfer_test.go b/connect/transfer_test.go index 9864fec..9f2a971 100644 --- a/connect/transfer_test.go +++ b/connect/transfer_test.go @@ -2,22 +2,21 @@ package connect import ( "context" - "testing" - "time" - mathrand "math/rand" - "fmt" - "crypto/hmac" + "crypto/hmac" "crypto/sha256" + "fmt" + mathrand "math/rand" "sync" + "testing" + "time" "google.golang.org/protobuf/proto" - "github.com/go-playground/assert/v2" + "github.com/go-playground/assert/v2" - "bringyour.com/protocol" + "github.com/bringyour/connect/protocol" ) - func TestSendReceiveSenderReset(t *testing.T) { // in this case two senders with the same client_id send after each other // The receiver should be able to reset using the new sequence_id @@ -27,7 +26,6 @@ func TestSendReceiveSenderReset(t *testing.T) { // number of messages n := 16 * 1024 - ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -57,11 +55,10 @@ func TestSendReceiveSenderReset(t *testing.T) { bReceiveTransport := NewReceiveGatewayTransport() provideModes := map[protocol.ProvideMode]bool{ - protocol.ProvideMode_Network: true, - } - + protocol.ProvideMode_Network: true, + } - clientSettingsA := DefaultClientSettings() + clientSettingsA := DefaultClientSettings() clientSettingsA.SendBufferSettings.SequenceBufferSize = 0 clientSettingsA.SendBufferSettings.AckBufferSize = 0 clientSettingsA.ReceiveBufferSettings.SequenceBufferSize = 0 @@ -79,10 +76,9 @@ func TestSendReceiveSenderReset(t *testing.T) { aRouteManager.UpdateTransport(aSendTransport, []Route{aSend}) aRouteManager.UpdateTransport(aReceiveTransport, []Route{aReceive}) - aContractManager.SetProvideModes(provideModes) - + aContractManager.SetProvideModes(provideModes) - clientSettingsB := DefaultClientSettings() + clientSettingsB := DefaultClientSettings() clientSettingsB.SendBufferSettings.SequenceBufferSize = 0 clientSettingsB.SendBufferSettings.AckBufferSize = 0 clientSettingsB.ReceiveBufferSettings.SequenceBufferSize = 0 @@ -102,7 +98,6 @@ func TestSendReceiveSenderReset(t *testing.T) { bContractManager.SetProvideModes(provideModes) - acks := make(chan error) receives := make(chan *protocol.SimpleMessage) @@ -120,8 +115,7 @@ func TestSendReceiveSenderReset(t *testing.T) { var receiveCount int var waitingReceiveCount int var receiveMessages map[string]bool - - + aReceive <- requireTransferFrameBytes( requireContractResultInitialPack( protocol.ProvideMode_Network, @@ -132,7 +126,6 @@ func TestSendReceiveSenderReset(t *testing.T) { ControlId, aClientId, ) - go func() { for i := 0; i < n; i += 1 { @@ -153,23 +146,23 @@ func TestSendReceiveSenderReset(t *testing.T) { receiveMessages = map[string]bool{} for receiveCount < n || ackCount < n { if receiveCount < n && waitingReceiveCount < receiveCount { - fmt.Printf("[0] waiting for %d/%d\n", receiveCount + 1, n) + fmt.Printf("[0] waiting for %d/%d\n", receiveCount+1, n) waitingReceiveCount = receiveCount } else if ackCount < n && waitingAckCount < ackCount { - fmt.Printf("[0] waiting for ack %d/%d\n", ackCount + 1, n) + fmt.Printf("[0] waiting for ack %d/%d\n", ackCount+1, n) } select { - case <- ctx.Done(): + case <-ctx.Done(): return - case message := <- receives: + case message := <-receives: receiveMessages[message.Content] = true assert.Equal(t, fmt.Sprintf("hi %d", receiveCount), message.Content) receiveCount += 1 - case err := <- acks: + case err := <-acks: assert.Equal(t, err, nil) ackCount += 1 - case <- time.After(timeout): + case <-time.After(timeout): t.Fatal("Timeout.") } } @@ -182,12 +175,10 @@ func TestSendReceiveSenderReset(t *testing.T) { assert.Equal(t, n, len(receiveMessages)) assert.Equal(t, n, ackCount) - a.Cancel() aRouteManager.RemoveTransport(aSendTransport) aRouteManager.RemoveTransport(aReceiveTransport) - a2 := NewClientWithDefaults(ctx, aClientId, NewNoContractClientOob()) a2RouteManager := a2.RouteManager() a2ContractManager := a2.ContractManager() @@ -202,7 +193,6 @@ func TestSendReceiveSenderReset(t *testing.T) { a2ContractManager.SetProvideModes(provideModes) - aReceive <- requireTransferFrameBytes( requireContractResultInitialPack( protocol.ProvideMode_Network, @@ -214,7 +204,6 @@ func TestSendReceiveSenderReset(t *testing.T) { aClientId, ) - go func() { for i := 0; i < n; i += 1 { message := &protocol.SimpleMessage{ @@ -234,23 +223,23 @@ func TestSendReceiveSenderReset(t *testing.T) { receiveMessages = map[string]bool{} for receiveCount < n || ackCount < n { if receiveCount < n && waitingReceiveCount < receiveCount { - fmt.Printf("[1] waiting for %d/%d\n", receiveCount + 1, n) + fmt.Printf("[1] waiting for %d/%d\n", receiveCount+1, n) waitingReceiveCount = receiveCount } else if ackCount < n && waitingAckCount < ackCount { - fmt.Printf("[1] waiting for ack %d/%d\n", ackCount + 1, n) + fmt.Printf("[1] waiting for ack %d/%d\n", ackCount+1, n) } select { - case <- ctx.Done(): + case <-ctx.Done(): return - case message := <- receives: + case message := <-receives: receiveMessages[message.Content] = true assert.Equal(t, fmt.Sprintf("hi %d", receiveCount), message.Content) receiveCount += 1 - case err := <- acks: + case err := <-acks: assert.Equal(t, err, nil) ackCount += 1 - case <- time.After(timeout): + case <-time.After(timeout): t.Fatal("Timeout.") } } @@ -270,7 +259,6 @@ func TestSendReceiveSenderReset(t *testing.T) { cancel() } - func createContractResultInitialPack( provideMode protocol.ProvideMode, provideSecretKey []byte, @@ -281,10 +269,10 @@ func createContractResultInitialPack( contractByteCount := 8 * 1024 * 1024 * 1024 storedContract := &protocol.StoredContract{ - ContractId: contractId.Bytes(), + ContractId: contractId.Bytes(), TransferByteCount: uint64(contractByteCount), - SourceId: sourceId.Bytes(), - DestinationId: destinationId.Bytes(), + SourceId: sourceId.Bytes(), + DestinationId: destinationId.Bytes(), } storedContractBytes, err := proto.Marshal(storedContract) if err != nil { @@ -296,8 +284,8 @@ func createContractResultInitialPack( message := &protocol.CreateContractResult{ Contract: &protocol.Contract{ StoredContractBytes: storedContractBytes, - StoredContractHmac: storedContractHmac, - ProvideMode: provideMode, + StoredContractHmac: storedContractHmac, + ProvideMode: provideMode, }, } @@ -309,17 +297,16 @@ func createContractResultInitialPack( messageId := NewId() sequenceId := NewId() pack := &protocol.Pack{ - MessageId: messageId.Bytes(), - SequenceId: sequenceId.Bytes(), + MessageId: messageId.Bytes(), + SequenceId: sequenceId.Bytes(), SequenceNumber: 0, - Head: true, - Frames: []*protocol.Frame{frame}, + Head: true, + Frames: []*protocol.Frame{frame}, } return ToFrame(pack) } - func requireContractResultInitialPack( provideMode protocol.ProvideMode, provideSecretKey []byte, @@ -333,11 +320,10 @@ func requireContractResultInitialPack( return frame } - func createTransferFrameBytes(frame *protocol.Frame, sourceId Id, destinationId Id) ([]byte, error) { transferFrame := &protocol.TransferFrame{ TransferPath: &protocol.TransferPath{ - SourceId: sourceId.Bytes(), + SourceId: sourceId.Bytes(), DestinationId: destinationId.Bytes(), // StreamId: DirectStreamId.Bytes(), }, @@ -347,7 +333,6 @@ func createTransferFrameBytes(frame *protocol.Frame, sourceId Id, destinationId return proto.Marshal(transferFrame) } - func requireTransferFrameBytes(frame *protocol.Frame, sourceId Id, destinationId Id) []byte { b, err := createTransferFrameBytes(frame, sourceId, destinationId) if err != nil { @@ -378,27 +363,25 @@ func requireTransferFrameBytes(frame *protocol.Frame, sourceId Id, destinationId return b } - - type conditioner struct { - ctx context.Context - fixedDelay time.Duration - randomDelay time.Duration - hold bool + ctx context.Context + fixedDelay time.Duration + randomDelay time.Duration + hold bool inversionWindow time.Duration - invertFraction float32 + invertFraction float32 lossProbability float32 - monitor *Monitor - mutex sync.Mutex + monitor *Monitor + mutex sync.Mutex } func newConditioner(ctx context.Context, in chan []byte) (*conditioner, chan []byte) { c := &conditioner{ - ctx: ctx, - fixedDelay: 0, - randomDelay: 0, + ctx: ctx, + fixedDelay: 0, + randomDelay: 0, lossProbability: 0, - monitor: NewMonitor(), + monitor: NewMonitor(), } out := make(chan []byte) go c.run(in, out) @@ -436,11 +419,11 @@ func (self *conditioner) run(in chan []byte, out chan []byte) { for { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return - case <- self.monitor.NotifyChannel(): + case <-self.monitor.NotifyChannel(): continue - case b, ok := <- in: + case b, ok := <-in: if !ok { return } @@ -453,33 +436,27 @@ func (self *conditioner) run(in chan []byte, out chan []byte) { if delay <= 0 { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return case out <- b: } } else { go func() { select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return - case <- time.After(delay): + case <-time.After(delay): } select { - case <- self.ctx.Done(): + case <-self.ctx.Done(): return case out <- b: } }() - } + } } } } - - // FIXME TestAckTimeout - - - - diff --git a/connect/transport.go b/connect/transport.go index a9c6816..000c970 100644 --- a/connect/transport.go +++ b/connect/transport.go @@ -1,27 +1,24 @@ package connect import ( - "context" - "time" - "net" - "fmt" - "bytes" + "bytes" + "context" + "fmt" + "net" + "time" - "github.com/gorilla/websocket" + "github.com/gorilla/websocket" - "github.com/golang/glog" + "github.com/golang/glog" - "bringyour.com/protocol" + "github.com/bringyour/connect/protocol" ) - const TransportBufferSize = 1 - // note that it is possible to have multiple transports for the same client destination // e.g. platform, p2p, and a bunch of extenders - // extenders are identified and credited with the platform by ip address // they forward to a special port, 8443, that whitelists their ip without rate limiting // when an extender gets an http message from a client, it always connects tcp to connect.bringyour.com:8443 @@ -30,452 +27,443 @@ const TransportBufferSize = 1 // rate limit using $proxy_protocol_addr https://www.nginx.com/blog/rate-limiting-nginx/ // add the source ip as the X-Extender header - type PlatformTransportSettings struct { - HttpConnectTimeout time.Duration - WsHandshakeTimeout time.Duration - AuthTimeout time.Duration - ReconnectTimeout time.Duration - PingTimeout time.Duration - WriteTimeout time.Duration - ReadTimeout time.Duration - TransportGenerator func()(sendTransport Transport, receiveTransport Transport) + HttpConnectTimeout time.Duration + WsHandshakeTimeout time.Duration + AuthTimeout time.Duration + ReconnectTimeout time.Duration + PingTimeout time.Duration + WriteTimeout time.Duration + ReadTimeout time.Duration + TransportGenerator func() (sendTransport Transport, receiveTransport Transport) } - func DefaultPlatformTransportSettings() *PlatformTransportSettings { - pingTimeout := 5 * time.Second - return &PlatformTransportSettings{ - HttpConnectTimeout: 2 * time.Second, - WsHandshakeTimeout: 2 * time.Second, - AuthTimeout: 2 * time.Second, - ReconnectTimeout: 5 * time.Second, - PingTimeout: pingTimeout, - WriteTimeout: 5 * time.Second, - ReadTimeout: 2 * pingTimeout, - } + pingTimeout := 5 * time.Second + return &PlatformTransportSettings{ + HttpConnectTimeout: 2 * time.Second, + WsHandshakeTimeout: 2 * time.Second, + AuthTimeout: 2 * time.Second, + ReconnectTimeout: 5 * time.Second, + PingTimeout: pingTimeout, + WriteTimeout: 5 * time.Second, + ReadTimeout: 2 * pingTimeout, + } } - type ClientAuth struct { - ByJwt string - // ClientId Id - InstanceId Id - AppVersion string + ByJwt string + // ClientId Id + InstanceId Id + AppVersion string } func (self *ClientAuth) ClientId() (Id, error) { - byJwt, err := ParseByJwtUnverified(self.ByJwt) - if err != nil { - return Id{}, err - } - return byJwt.ClientId, nil + byJwt, err := ParseByJwtUnverified(self.ByJwt) + if err != nil { + return Id{}, err + } + return byJwt.ClientId, nil } - // (ctx, network, address) type DialContextFunc func(ctx context.Context, network string, address string) (net.Conn, error) - type PlatformTransport struct { - ctx context.Context - cancel context.CancelFunc + ctx context.Context + cancel context.CancelFunc - platformUrl string - auth *ClientAuth - dialContextGen func()(DialContextFunc) - settings *PlatformTransportSettings + platformUrl string + auth *ClientAuth + dialContextGen func() DialContextFunc + settings *PlatformTransportSettings - routeManager *RouteManager + routeManager *RouteManager } func NewPlatformTransportWithDefaults( - ctx context.Context, - platformUrl string, - auth *ClientAuth, - routeManager *RouteManager, + ctx context.Context, + platformUrl string, + auth *ClientAuth, + routeManager *RouteManager, ) *PlatformTransport { - return NewPlatformTransportWithDefaultDialer( - ctx, - platformUrl, - auth, - routeManager, - DefaultPlatformTransportSettings(), - ) + return NewPlatformTransportWithDefaultDialer( + ctx, + platformUrl, + auth, + routeManager, + DefaultPlatformTransportSettings(), + ) } func NewPlatformTransportWithDefaultDialer( - ctx context.Context, - platformUrl string, - auth *ClientAuth, - routeManager *RouteManager, - settings *PlatformTransportSettings, + ctx context.Context, + platformUrl string, + auth *ClientAuth, + routeManager *RouteManager, + settings *PlatformTransportSettings, ) *PlatformTransport { - dialContextGen := func()(DialContextFunc) { - dialer := &net.Dialer{ - Timeout: settings.HttpConnectTimeout, - } - return dialer.DialContext - } - - return NewPlatformTransport( - ctx, - platformUrl, - auth, - dialContextGen, - settings, - routeManager, - ) + dialContextGen := func() DialContextFunc { + dialer := &net.Dialer{ + Timeout: settings.HttpConnectTimeout, + } + return dialer.DialContext + } + + return NewPlatformTransport( + ctx, + platformUrl, + auth, + dialContextGen, + settings, + routeManager, + ) } func NewPlatformTransportWithExtender( - ctx context.Context, - extenderUrl string, - platformUrl string, - auth *ClientAuth, - settings *PlatformTransportSettings, - routeManager *RouteManager, + ctx context.Context, + extenderUrl string, + platformUrl string, + auth *ClientAuth, + settings *PlatformTransportSettings, + routeManager *RouteManager, ) *PlatformTransport { - return NewPlatformTransport( - ctx, - platformUrl, - auth, - NewExtenderDialContextGenerator(extenderUrl, settings), - settings, - routeManager, - ) + return NewPlatformTransport( + ctx, + platformUrl, + auth, + NewExtenderDialContextGenerator(extenderUrl, settings), + settings, + routeManager, + ) } func NewPlatformTransport( - ctx context.Context, - platformUrl string, - auth *ClientAuth, - dialContextGen func()(DialContextFunc), - settings *PlatformTransportSettings, - routeManager *RouteManager, + ctx context.Context, + platformUrl string, + auth *ClientAuth, + dialContextGen func() DialContextFunc, + settings *PlatformTransportSettings, + routeManager *RouteManager, ) *PlatformTransport { - cancelCtx, cancel := context.WithCancel(ctx) - transport := &PlatformTransport{ - ctx: cancelCtx, - cancel: cancel, - platformUrl: platformUrl, - auth: auth, - dialContextGen: dialContextGen, - settings: settings, - routeManager: routeManager, - } - go transport.run() - return transport + cancelCtx, cancel := context.WithCancel(ctx) + transport := &PlatformTransport{ + ctx: cancelCtx, + cancel: cancel, + platformUrl: platformUrl, + auth: auth, + dialContextGen: dialContextGen, + settings: settings, + routeManager: routeManager, + } + go transport.run() + return transport } func (self *PlatformTransport) run() { - // connect and update route manager for this transport - defer self.cancel() - - clientId, _ := self.auth.ClientId() - - authBytes, err := EncodeFrame(&protocol.Auth{ - ByJwt: self.auth.ByJwt, - AppVersion: self.auth.AppVersion, - InstanceId: self.auth.InstanceId.Bytes(), - }) - if err != nil { - return - } - - for { - wsDialer := &websocket.Dialer{ - NetDialContext: self.dialContextGen(), - HandshakeTimeout: self.settings.WsHandshakeTimeout, - } - - ws, err := func()(*websocket.Conn, error) { - ws, _, err := wsDialer.DialContext(self.ctx, self.platformUrl, nil) - if err != nil { - return nil, err - } - - success := false - defer func() { - if !success { - ws.Close() - } - }() - - ws.SetWriteDeadline(time.Now().Add(self.settings.AuthTimeout)) - if err := ws.WriteMessage(websocket.BinaryMessage, authBytes); err != nil { - return nil, err - } - ws.SetReadDeadline(time.Now().Add(self.settings.AuthTimeout)) - if messageType, message, err := ws.ReadMessage(); err != nil { - return nil, err - } else { - // verify the auth echo - switch messageType { - case websocket.BinaryMessage: - if !bytes.Equal(authBytes, message) { - return nil, fmt.Errorf("Auth response error: bad bytes.") - } - default: - return nil, fmt.Errorf("Auth response error.") - } - } - - success = true - return ws, nil - }() - if err != nil { - glog.Infof("[t]auth error %s = %s\n", clientId, err) - select { - case <- self.ctx.Done(): - return - case <- time.After(self.settings.ReconnectTimeout): - continue - } - } - - c := func() { - defer ws.Close() - - handleCtx, handleCancel := context.WithCancel(self.ctx) - defer handleCancel() - - send := make(chan []byte, TransportBufferSize) - receive := make(chan []byte, TransportBufferSize) - - // the platform can route any destination, - // since every client has a platform transport - var sendTransport Transport - var receiveTransport Transport - if self.settings.TransportGenerator != nil { - sendTransport, receiveTransport = self.settings.TransportGenerator() - } else { - sendTransport = NewSendGatewayTransport() - receiveTransport = NewReceiveGatewayTransport() - } - - self.routeManager.UpdateTransport(sendTransport, []Route{send}) - self.routeManager.UpdateTransport(receiveTransport, []Route{receive}) - - defer func() { - self.routeManager.RemoveTransport(sendTransport) - self.routeManager.RemoveTransport(receiveTransport) - - // note `send` is not closed. This channel is left open. - // it used to be closed after a delay, but it is not needed to close it. - }() - - go func() { - defer handleCancel() - - for { - select { - case <- handleCtx.Done(): - return - case message, ok := <- send: - if !ok { - return - } - - ws.SetWriteDeadline(time.Now().Add(self.settings.WriteTimeout)) - if err := ws.WriteMessage(websocket.BinaryMessage, message); err != nil { - // note that for websocket a dealine timeout cannot be recovered - glog.V(2).Infof("[ts]%s-> error = %s\n", clientId, err) - return - } - glog.V(2).Infof("[ts]%s->\n", clientId) - case <- time.After(self.settings.PingTimeout): - ws.SetWriteDeadline(time.Now().Add(self.settings.WriteTimeout)) - if err := ws.WriteMessage(websocket.BinaryMessage, make([]byte, 0)); err != nil { - // note that for websocket a dealine timeout cannot be recovered - return - } - } - } - }() - - go func() { - defer func() { - handleCancel() - close(receive) - }() - - for { - select { - case <- handleCtx.Done(): - return - default: - } - - ws.SetReadDeadline(time.Now().Add(self.settings.ReadTimeout)) - messageType, message, err := ws.ReadMessage() - if err != nil { - glog.V(2).Infof("[tr]%s<- error = %s\n", clientId, err) - return - } - - switch messageType { - case websocket.BinaryMessage: - if 0 == len(message) { - // ping - glog.V(2).Infof("[tr]ping %s<-\n", clientId) - continue - } - - select { - case <- handleCtx.Done(): - return - case receive <- message: - glog.V(2).Infof("[tr]%s<-\n", clientId) - case <- time.After(self.settings.ReadTimeout): - glog.Infof("[tr]drop %s<-\n", clientId) - } - } - } - }() - - select { - case <- handleCtx.Done(): - } - } - if glog.V(2) { - Trace(fmt.Sprintf("[t]connect %s", clientId), c) - } else { - c() - } - - select { - case <- self.ctx.Done(): - return - case <- time.After(self.settings.ReconnectTimeout): - } - } + // connect and update route manager for this transport + defer self.cancel() + + clientId, _ := self.auth.ClientId() + + authBytes, err := EncodeFrame(&protocol.Auth{ + ByJwt: self.auth.ByJwt, + AppVersion: self.auth.AppVersion, + InstanceId: self.auth.InstanceId.Bytes(), + }) + if err != nil { + return + } + + for { + wsDialer := &websocket.Dialer{ + NetDialContext: self.dialContextGen(), + HandshakeTimeout: self.settings.WsHandshakeTimeout, + } + + ws, err := func() (*websocket.Conn, error) { + ws, _, err := wsDialer.DialContext(self.ctx, self.platformUrl, nil) + if err != nil { + return nil, err + } + + success := false + defer func() { + if !success { + ws.Close() + } + }() + + ws.SetWriteDeadline(time.Now().Add(self.settings.AuthTimeout)) + if err := ws.WriteMessage(websocket.BinaryMessage, authBytes); err != nil { + return nil, err + } + ws.SetReadDeadline(time.Now().Add(self.settings.AuthTimeout)) + if messageType, message, err := ws.ReadMessage(); err != nil { + return nil, err + } else { + // verify the auth echo + switch messageType { + case websocket.BinaryMessage: + if !bytes.Equal(authBytes, message) { + return nil, fmt.Errorf("Auth response error: bad bytes.") + } + default: + return nil, fmt.Errorf("Auth response error.") + } + } + + success = true + return ws, nil + }() + if err != nil { + glog.Infof("[t]auth error %s = %s\n", clientId, err) + select { + case <-self.ctx.Done(): + return + case <-time.After(self.settings.ReconnectTimeout): + continue + } + } + + c := func() { + defer ws.Close() + + handleCtx, handleCancel := context.WithCancel(self.ctx) + defer handleCancel() + + send := make(chan []byte, TransportBufferSize) + receive := make(chan []byte, TransportBufferSize) + + // the platform can route any destination, + // since every client has a platform transport + var sendTransport Transport + var receiveTransport Transport + if self.settings.TransportGenerator != nil { + sendTransport, receiveTransport = self.settings.TransportGenerator() + } else { + sendTransport = NewSendGatewayTransport() + receiveTransport = NewReceiveGatewayTransport() + } + + self.routeManager.UpdateTransport(sendTransport, []Route{send}) + self.routeManager.UpdateTransport(receiveTransport, []Route{receive}) + + defer func() { + self.routeManager.RemoveTransport(sendTransport) + self.routeManager.RemoveTransport(receiveTransport) + + // note `send` is not closed. This channel is left open. + // it used to be closed after a delay, but it is not needed to close it. + }() + + go func() { + defer handleCancel() + + for { + select { + case <-handleCtx.Done(): + return + case message, ok := <-send: + if !ok { + return + } + + ws.SetWriteDeadline(time.Now().Add(self.settings.WriteTimeout)) + if err := ws.WriteMessage(websocket.BinaryMessage, message); err != nil { + // note that for websocket a dealine timeout cannot be recovered + glog.V(2).Infof("[ts]%s-> error = %s\n", clientId, err) + return + } + glog.V(2).Infof("[ts]%s->\n", clientId) + case <-time.After(self.settings.PingTimeout): + ws.SetWriteDeadline(time.Now().Add(self.settings.WriteTimeout)) + if err := ws.WriteMessage(websocket.BinaryMessage, make([]byte, 0)); err != nil { + // note that for websocket a dealine timeout cannot be recovered + return + } + } + } + }() + + go func() { + defer func() { + handleCancel() + close(receive) + }() + + for { + select { + case <-handleCtx.Done(): + return + default: + } + + ws.SetReadDeadline(time.Now().Add(self.settings.ReadTimeout)) + messageType, message, err := ws.ReadMessage() + if err != nil { + glog.V(2).Infof("[tr]%s<- error = %s\n", clientId, err) + return + } + + switch messageType { + case websocket.BinaryMessage: + if 0 == len(message) { + // ping + glog.V(2).Infof("[tr]ping %s<-\n", clientId) + continue + } + + select { + case <-handleCtx.Done(): + return + case receive <- message: + glog.V(2).Infof("[tr]%s<-\n", clientId) + case <-time.After(self.settings.ReadTimeout): + glog.Infof("[tr]drop %s<-\n", clientId) + } + } + } + }() + + select { + case <-handleCtx.Done(): + } + } + if glog.V(2) { + Trace(fmt.Sprintf("[t]connect %s", clientId), c) + } else { + c() + } + + select { + case <-self.ctx.Done(): + return + case <-time.After(self.settings.ReconnectTimeout): + } + } } func (self *PlatformTransport) Close() { - self.cancel() + self.cancel() } - // an extender uses an independent url that is hard-coded to forward to the platform // the `platformUrl` here must match the hard coded url in the extender, which is // done by using a prior vetted extender // The connection to the platform is end-to-end encrypted with TLS, // using the hostname from `platformUrl` - func NewExtenderDialContextGenerator( - extenderUrl string, - settings *PlatformTransportSettings, -) func()(DialContextFunc) { - return func()(DialContextFunc) { - return func( - ctx context.Context, - network string, - address string, - ) (net.Conn, error) { - dialer := &net.Dialer{ - Timeout: settings.HttpConnectTimeout, - } - - wsDialer := &websocket.Dialer{ - NetDialContext: dialer.DialContext, - HandshakeTimeout: settings.WsHandshakeTimeout, - } - - ws, _, err := wsDialer.DialContext(ctx, extenderUrl, nil) - if err != nil { - return nil, err - } - - return newWsForwardingConn(ws, settings), nil - } - } + extenderUrl string, + settings *PlatformTransportSettings, +) func() DialContextFunc { + return func() DialContextFunc { + return func( + ctx context.Context, + network string, + address string, + ) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: settings.HttpConnectTimeout, + } + + wsDialer := &websocket.Dialer{ + NetDialContext: dialer.DialContext, + HandshakeTimeout: settings.WsHandshakeTimeout, + } + + ws, _, err := wsDialer.DialContext(ctx, extenderUrl, nil) + if err != nil { + return nil, err + } + + return newWsForwardingConn(ws, settings), nil + } + } } - // conforms to `net.Conn` type wsForwardingConn struct { - ws *websocket.Conn - readBuffer []byte - settings *PlatformTransportSettings + ws *websocket.Conn + readBuffer []byte + settings *PlatformTransportSettings } func newWsForwardingConn(ws *websocket.Conn, settings *PlatformTransportSettings) *wsForwardingConn { - return &wsForwardingConn{ - ws: ws, - readBuffer: make([]byte, 0), - settings: settings, - } + return &wsForwardingConn{ + ws: ws, + readBuffer: make([]byte, 0), + settings: settings, + } } func (self *wsForwardingConn) Read(b []byte) (int, error) { - i := min(len(b), len(self.readBuffer)) - if 0 < i { - copy(b, self.readBuffer[:i]) - self.readBuffer = self.readBuffer[i:] - } - for i < len(b) { - self.ws.SetReadDeadline(time.Now().Add(self.settings.ReadTimeout)) - messageType, message, err := self.ws.ReadMessage() - if err != nil { - return i, err - } - switch messageType { - case websocket.BinaryMessage: - if 0 == len(message) { - // ping - continue - } - - j := min(len(b) - i, len(message)) - copy(b[i:], message[:j]) - i += j - if j < len(message) { - self.readBuffer = append(self.readBuffer, message[j:]...) - } - } - } - return i, nil + i := min(len(b), len(self.readBuffer)) + if 0 < i { + copy(b, self.readBuffer[:i]) + self.readBuffer = self.readBuffer[i:] + } + for i < len(b) { + self.ws.SetReadDeadline(time.Now().Add(self.settings.ReadTimeout)) + messageType, message, err := self.ws.ReadMessage() + if err != nil { + return i, err + } + switch messageType { + case websocket.BinaryMessage: + if 0 == len(message) { + // ping + continue + } + + j := min(len(b)-i, len(message)) + copy(b[i:], message[:j]) + i += j + if j < len(message) { + self.readBuffer = append(self.readBuffer, message[j:]...) + } + } + } + return i, nil } func (self *wsForwardingConn) Write(b []byte) (int, error) { - self.ws.SetWriteDeadline(time.Now().Add(self.settings.WriteTimeout)) - err := self.ws.WriteMessage(websocket.BinaryMessage, b) - if err != nil { - // note that for websocket a dealine timeout cannot be recovered - return 0, err - } - return len(b), nil + self.ws.SetWriteDeadline(time.Now().Add(self.settings.WriteTimeout)) + err := self.ws.WriteMessage(websocket.BinaryMessage, b) + if err != nil { + // note that for websocket a dealine timeout cannot be recovered + return 0, err + } + return len(b), nil } func (self *wsForwardingConn) Close() error { - return self.ws.Close() + return self.ws.Close() } func (self *wsForwardingConn) LocalAddr() net.Addr { - return self.ws.LocalAddr() + return self.ws.LocalAddr() } func (self *wsForwardingConn) RemoteAddr() net.Addr { - return self.ws.RemoteAddr() + return self.ws.RemoteAddr() } func (self *wsForwardingConn) SetDeadline(t time.Time) error { - if err := self.ws.SetReadDeadline(t); err != nil { - return err - } - if err := self.ws.SetWriteDeadline(t); err != nil { - return err - } - return nil + if err := self.ws.SetReadDeadline(t); err != nil { + return err + } + if err := self.ws.SetWriteDeadline(t); err != nil { + return err + } + return nil } func (self *wsForwardingConn) SetReadDeadline(t time.Time) error { - return self.ws.SetReadDeadline(t) + return self.ws.SetReadDeadline(t) } func (self *wsForwardingConn) SetWriteDeadline(t time.Time) error { - return self.ws.SetWriteDeadline(t) + return self.ws.SetWriteDeadline(t) } - diff --git a/connectctl/go.mod b/connectctl/go.mod deleted file mode 100644 index 4c9783f..0000000 --- a/connectctl/go.mod +++ /dev/null @@ -1,22 +0,0 @@ -module bringyour.com/connectctl - -go 1.22.0 - -replace bringyour.com/connect v0.0.0 => ../connect - -replace bringyour.com/protocol v0.0.0 => ../protocol/build/bringyour.com/protocol - -require ( - bringyour.com/connect v0.0.0 - bringyour.com/protocol v0.0.0 - github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 - github.com/golang-jwt/jwt/v5 v5.2.0 -) - -require ( - github.com/google/gopacket v1.1.19 // indirect - github.com/gorilla/websocket v1.5.0 // indirect - github.com/oklog/ulid/v2 v2.1.0 // indirect - golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect - google.golang.org/protobuf v1.31.0 // indirect -) diff --git a/connectctl/go.sum b/connectctl/go.sum deleted file mode 100644 index 1600884..0000000 --- a/connectctl/go.sum +++ /dev/null @@ -1,35 +0,0 @@ -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= -github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= -github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU= -github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= -github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/connectctl/main.go b/connectctl/main.go index de109ae..51bedb7 100644 --- a/connectctl/main.go +++ b/connectctl/main.go @@ -1,58 +1,56 @@ package main - import ( - "context" - "fmt" - "os" - // "os/exec" - // "path/filepath" - // "encoding/json" - "time" - // "strings" - // "math" - // "reflect" - // "sort" - // "syscall" - // "os/signal" - // "errors" - // "regexp" - "io" - "net/http" - "log" - "encoding/json" - // "encoding/base64" - "bytes" - - // "golang.org/x/exp/maps" - - gojwt "github.com/golang-jwt/jwt/v5" - - "github.com/docopt/docopt-go" - - "bringyour.com/connect" - "bringyour.com/protocol" + "context" + "fmt" + "os" + + // "os/exec" + // "path/filepath" + // "encoding/json" + "time" + // "strings" + // "math" + // "reflect" + // "sort" + // "syscall" + // "os/signal" + // "errors" + // "regexp" + "encoding/json" + "io" + "log" + "net/http" + + // "encoding/base64" + "bytes" + + // "golang.org/x/exp/maps" + + gojwt "github.com/golang-jwt/jwt/v5" + + "github.com/docopt/docopt-go" + + "github.com/bringyour/connect/connect" + "github.com/bringyour/connect/protocol" ) - const ConnectCtlVersion = "0.0.1" const DefaultApiUrl = "https://api.bringyour.com" const DefaultConnectUrl = "wss://connect.bringyour.com" - var Out *log.Logger var Err *log.Logger func init() { - Out = log.New(os.Stdout, "", 0) - Err = log.New(os.Stderr, "", log.Ldate | log.Ltime | log.Lshortfile) + Out = log.New(os.Stdout, "", 0) + Err = log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lshortfile) } - func main() { - usage := fmt.Sprintf( - `Connect control. + usage := fmt.Sprintf( + `Connect control. The default urls are: api_url: %s @@ -99,625 +97,600 @@ Options: --destination_id= Destination client_id --message_count= Print this many messages then exit. --instance_id= Set the client instance id.`, - DefaultApiUrl, - DefaultConnectUrl, - ) - - opts, err := docopt.ParseArgs(usage, os.Args[1:], ConnectCtlVersion) - if err != nil { - panic(err) - } - - if createNetwork_, _ := opts.Bool("create-network"); createNetwork_ { - createNetwork(opts) - } else if verifySend_, _ := opts.Bool("verify-send"); verifySend_ { - verifySend(opts) - } else if verifyNetwork_, _ := opts.Bool("verify-network"); verifyNetwork_ { - verifyNetwork(opts) - } else if loginNetwork_, _ := opts.Bool("login-network"); loginNetwork_ { - loginNetwork(opts) - } else if clientId_, _ := opts.Bool("client-id"); clientId_ { - clientId(opts) - } else if send_, _ := opts.Bool("send"); send_ { - send(opts) - } else if sink_, _ := opts.Bool("sink"); sink_ { - sink(opts) - } + DefaultApiUrl, + DefaultConnectUrl, + ) + + opts, err := docopt.ParseArgs(usage, os.Args[1:], ConnectCtlVersion) + if err != nil { + panic(err) + } + + if createNetwork_, _ := opts.Bool("create-network"); createNetwork_ { + createNetwork(opts) + } else if verifySend_, _ := opts.Bool("verify-send"); verifySend_ { + verifySend(opts) + } else if verifyNetwork_, _ := opts.Bool("verify-network"); verifyNetwork_ { + verifyNetwork(opts) + } else if loginNetwork_, _ := opts.Bool("login-network"); loginNetwork_ { + loginNetwork(opts) + } else if clientId_, _ := opts.Bool("client-id"); clientId_ { + clientId(opts) + } else if send_, _ := opts.Bool("send"); send_ { + send(opts) + } else if sink_, _ := opts.Bool("sink"); sink_ { + sink(opts) + } } - func printResult(result map[string]any) { - expandByJwt(result) - - out, err := json.MarshalIndent(result, "", " ") - if err != nil { - panic(err) - } - fmt.Printf("%s\n", out) + expandByJwt(result) + + out, err := json.MarshalIndent(result, "", " ") + if err != nil { + panic(err) + } + fmt.Printf("%s\n", out) } func expandByJwt(result map[string]any) { - if jwt, ok := result["by_jwt"]; ok { - claims := gojwt.MapClaims{} - gojwt.NewParser().ParseUnverified(jwt.(string), claims) - - for claimKey, claimValue := range claims { - result[fmt.Sprintf("by_jwt_%s", claimKey)] = claimValue - } - } - if jwt, ok := result["by_client_jwt"]; ok { - claims := gojwt.MapClaims{} - gojwt.NewParser().ParseUnverified(jwt.(string), claims) - - for claimKey, claimValue := range claims { - result[fmt.Sprintf("by_client_jwt_%s", claimKey)] = claimValue - } - } - for _, value := range result { - if subResult, ok := value.(map[string]any); ok { - expandByJwt(subResult) - } - } + if jwt, ok := result["by_jwt"]; ok { + claims := gojwt.MapClaims{} + gojwt.NewParser().ParseUnverified(jwt.(string), claims) + + for claimKey, claimValue := range claims { + result[fmt.Sprintf("by_jwt_%s", claimKey)] = claimValue + } + } + if jwt, ok := result["by_client_jwt"]; ok { + claims := gojwt.MapClaims{} + gojwt.NewParser().ParseUnverified(jwt.(string), claims) + + for claimKey, claimValue := range claims { + result[fmt.Sprintf("by_client_jwt_%s", claimKey)] = claimValue + } + } + for _, value := range result { + if subResult, ok := value.(map[string]any); ok { + expandByJwt(subResult) + } + } } - func createNetwork(opts docopt.Opts) { - apiUrl, err := opts.String("--api_url") - if err != nil { - apiUrl = DefaultApiUrl - } - - networkName, _ := opts.String("--network_name") + apiUrl, err := opts.String("--api_url") + if err != nil { + apiUrl = DefaultApiUrl + } - userName, _ := opts.String("--user_name") + networkName, _ := opts.String("--network_name") - userAuth, _ := opts.String("--user_auth") + userName, _ := opts.String("--user_name") - password, _ := opts.String("--password") + userAuth, _ := opts.String("--user_auth") - timeout := 5 * time.Second + password, _ := opts.String("--password") + timeout := 5 * time.Second - // /auth/network-create - args := map[string]any{} - args["user_name"] = userName - args["user_auth"] = userAuth - args["password"] = password - args["network_name"] = networkName - args["terms"] = true + // /auth/network-create + args := map[string]any{} + args["user_name"] = userName + args["user_auth"] = userAuth + args["password"] = password + args["network_name"] = networkName + args["terms"] = true - reqBody, err := json.Marshal(args) + reqBody, err := json.Marshal(args) - // fmt.Printf("request: %s\n", reqBody) + // fmt.Printf("request: %s\n", reqBody) - req, err := http.NewRequest( - "POST", - fmt.Sprintf("%s/auth/network-create", apiUrl), - bytes.NewReader(reqBody), - ) - if err != nil { - return - } - req.Header.Set("Content-Type", "application/json") + req, err := http.NewRequest( + "POST", + fmt.Sprintf("%s/auth/network-create", apiUrl), + bytes.NewReader(reqBody), + ) + if err != nil { + return + } + req.Header.Set("Content-Type", "application/json") - client := &http.Client{ - Timeout: timeout, - } + client := &http.Client{ + Timeout: timeout, + } - res, err := client.Do(req) - if err != nil { - panic(err) - } - resBody, err := io.ReadAll(res.Body) - if err != nil { - panic(err) - } + res, err := client.Do(req) + if err != nil { + panic(err) + } + resBody, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + } - // fmt.Printf("response: %s\n", resBody) + // fmt.Printf("response: %s\n", resBody) - result := map[string]any{} - err = json.Unmarshal(resBody, &result) - if err != nil { - panic(err) - } + result := map[string]any{} + err = json.Unmarshal(resBody, &result) + if err != nil { + panic(err) + } - printResult(result) + printResult(result) } - func verifySend(opts docopt.Opts) { - apiUrl, err := opts.String("--api_url") - if err != nil { - apiUrl = DefaultApiUrl - } - - userAuth, _ := opts.String("--user_auth") - - timeout := 5 * time.Second - - - // /auth/verify-send - args := map[string]any{} - args["user_auth"] = userAuth - - reqBody, err := json.Marshal(args) - - req, err := http.NewRequest( - "POST", - fmt.Sprintf("%s/auth/verify-send", apiUrl), - bytes.NewReader(reqBody), - ) - if err != nil { - return - } - req.Header.Set("Content-Type", "application/json") - - client := &http.Client{ - Timeout: timeout, - } - - res, err := client.Do(req) - if err != nil { - panic(err) - return - } - resBody, err := io.ReadAll(res.Body) - if err != nil { - panic(err) - return - } - - result := map[string]any{} - err = json.Unmarshal(resBody, &result) - if err != nil { - panic(err) - return - } - - printResult(result) + apiUrl, err := opts.String("--api_url") + if err != nil { + apiUrl = DefaultApiUrl + } + + userAuth, _ := opts.String("--user_auth") + + timeout := 5 * time.Second + + // /auth/verify-send + args := map[string]any{} + args["user_auth"] = userAuth + + reqBody, err := json.Marshal(args) + + req, err := http.NewRequest( + "POST", + fmt.Sprintf("%s/auth/verify-send", apiUrl), + bytes.NewReader(reqBody), + ) + if err != nil { + return + } + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{ + Timeout: timeout, + } + + res, err := client.Do(req) + if err != nil { + panic(err) + return + } + resBody, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + return + } + + result := map[string]any{} + err = json.Unmarshal(resBody, &result) + if err != nil { + panic(err) + return + } + + printResult(result) } - func verifyNetwork(opts docopt.Opts) { - apiUrl, err := opts.String("--api_url") - if err != nil { - apiUrl = DefaultApiUrl - } - - userAuth, _ := opts.String("--user_auth") - - verifyCode, _ := opts.String("--code") - - timeout := 5 * time.Second - - - // /auth/verify - args := map[string]any{} - args["user_auth"] = userAuth - args["verify_code"] = verifyCode - - reqBody, err := json.Marshal(args) - - req, err := http.NewRequest( - "POST", - fmt.Sprintf("%s/auth/verify", apiUrl), - bytes.NewReader(reqBody), - ) - if err != nil { - return - } - req.Header.Set("Content-Type", "application/json") - - client := &http.Client{ - Timeout: timeout, - } - - res, err := client.Do(req) - if err != nil { - panic(err) - return - } - resBody, err := io.ReadAll(res.Body) - if err != nil { - panic(err) - return - } - - result := map[string]any{} - err = json.Unmarshal(resBody, &result) - if err != nil { - panic(err) - return - } - - printResult(result) + apiUrl, err := opts.String("--api_url") + if err != nil { + apiUrl = DefaultApiUrl + } + + userAuth, _ := opts.String("--user_auth") + + verifyCode, _ := opts.String("--code") + + timeout := 5 * time.Second + + // /auth/verify + args := map[string]any{} + args["user_auth"] = userAuth + args["verify_code"] = verifyCode + + reqBody, err := json.Marshal(args) + + req, err := http.NewRequest( + "POST", + fmt.Sprintf("%s/auth/verify", apiUrl), + bytes.NewReader(reqBody), + ) + if err != nil { + return + } + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{ + Timeout: timeout, + } + + res, err := client.Do(req) + if err != nil { + panic(err) + return + } + resBody, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + return + } + + result := map[string]any{} + err = json.Unmarshal(resBody, &result) + if err != nil { + panic(err) + return + } + + printResult(result) } - func loginNetwork(opts docopt.Opts) { - apiUrl, err := opts.String("--api_url") - if err != nil { - apiUrl = DefaultApiUrl - } - - userAuth, _ := opts.String("--user_auth") - - password, _ := opts.String("--password") - - timeout := 5 * time.Second - - - // /auth/login-with-password - args := map[string]any{} - args["user_auth"] = userAuth - args["password"] = password - - reqBody, err := json.Marshal(args) - - - req, err := http.NewRequest( - "POST", - fmt.Sprintf("%s/auth/login-with-password", apiUrl), - bytes.NewReader(reqBody), - ) - if err != nil { - return - } - req.Header.Set("Content-Type", "application/json") - - client := &http.Client{ - Timeout: timeout, - } - - res, err := client.Do(req) - if err != nil { - panic(err) - return - } - resBody, err := io.ReadAll(res.Body) - if err != nil { - panic(err) - return - } - - result := map[string]any{} - err = json.Unmarshal(resBody, &result) - if err != nil { - panic(err) - return - } - - printResult(result) + apiUrl, err := opts.String("--api_url") + if err != nil { + apiUrl = DefaultApiUrl + } + + userAuth, _ := opts.String("--user_auth") + + password, _ := opts.String("--password") + + timeout := 5 * time.Second + + // /auth/login-with-password + args := map[string]any{} + args["user_auth"] = userAuth + args["password"] = password + + reqBody, err := json.Marshal(args) + + req, err := http.NewRequest( + "POST", + fmt.Sprintf("%s/auth/login-with-password", apiUrl), + bytes.NewReader(reqBody), + ) + if err != nil { + return + } + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{ + Timeout: timeout, + } + + res, err := client.Do(req) + if err != nil { + panic(err) + return + } + resBody, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + return + } + + result := map[string]any{} + err = json.Unmarshal(resBody, &result) + if err != nil { + panic(err) + return + } + + printResult(result) } - // use the given jwt to generate a new jwt with a new client id func clientId(opts docopt.Opts) { - apiUrl, err := opts.String("--api_url") - if err != nil { - apiUrl = DefaultApiUrl - } - - jwt, _ := opts.String("--jwt") - - bearer := []byte(jwt) - - timeout := 5 * time.Second - - - // /network/auth-client - args := map[string]any{} - args["description"] = "" - args["device_spec"] = "" - - reqBody, err := json.Marshal(args) - - // fmt.Printf("request: %s\n", reqBody) - - req, err := http.NewRequest( - "POST", - fmt.Sprintf("%s/network/auth-client", apiUrl), - bytes.NewReader(reqBody), - ) - if err != nil { - return - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", bearer)) - - client := &http.Client{ - Timeout: timeout, - } - - res, err := client.Do(req) - if err != nil { - panic(err) - return - } - resBody, err := io.ReadAll(res.Body) - if err != nil { - panic(err) - return - } - - fmt.Printf("response: %s\n", resBody) - - result := map[string]any{} - err = json.Unmarshal(resBody, &result) - if err != nil { - panic(err) - return - } - - printResult(result) + apiUrl, err := opts.String("--api_url") + if err != nil { + apiUrl = DefaultApiUrl + } + + jwt, _ := opts.String("--jwt") + + bearer := []byte(jwt) + + timeout := 5 * time.Second + + // /network/auth-client + args := map[string]any{} + args["description"] = "" + args["device_spec"] = "" + + reqBody, err := json.Marshal(args) + + // fmt.Printf("request: %s\n", reqBody) + + req, err := http.NewRequest( + "POST", + fmt.Sprintf("%s/network/auth-client", apiUrl), + bytes.NewReader(reqBody), + ) + if err != nil { + return + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", bearer)) + + client := &http.Client{ + Timeout: timeout, + } + + res, err := client.Do(req) + if err != nil { + panic(err) + return + } + resBody, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + return + } + + fmt.Printf("response: %s\n", resBody) + + result := map[string]any{} + err = json.Unmarshal(resBody, &result) + if err != nil { + panic(err) + return + } + + printResult(result) } - func send(opts docopt.Opts) { - jwt, _ := opts.String("--jwt") - - var clientId connect.Id - - claims := gojwt.MapClaims{} - gojwt.NewParser().ParseUnverified(jwt, claims) - - jwtClientId, ok := claims["client_id"] - if !ok { - fmt.Printf("JWT does not have a client_id.\n") - return - } - switch v := jwtClientId.(type) { - case string: - var err error - clientId, err = connect.ParseId(v) - if err != nil { - fmt.Printf("JWT has invalid client_id (%s).\n", err) - return - } - default: - fmt.Printf("JWT has invalid client_id (%T).\n", v) - return - } - - fmt.Printf("client_id: %s\n", clientId.String()) - - connectUrl, err := opts.String("--connect_url") - if err != nil { - connectUrl = DefaultConnectUrl - } - - destinationIdStr, _ := opts.String("--destination_id") - destinationId, err := connect.ParseId(destinationIdStr) - if err != nil { - fmt.Printf("Invalid destination_id (%s).\n", err) - return - } - - instanceIdStr, err := opts.String("--instance_id") - var instanceId connect.Id - if err == nil { - instanceId, err = connect.ParseId(instanceIdStr) - if err != nil { - fmt.Printf("Invalid instance_id (%s).\n", err) - return - } - } else { - instanceId = connect.NewId() - } - - fmt.Printf("instance_id: %s\n", instanceId.String()) - - - messageContent, _ := opts.String("") - - messageCount, err := opts.Int("--message_count") - if err != nil { - messageCount = 1 - } - - // need at least one. Use more for testing. - transportCount := 4 - - timeout := 30 * time.Second - - - cancelCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - - client := connect.NewClientWithDefaults( - cancelCtx, - clientId, - ) - defer client.Close() - - - // client.SetInstanceId(instanceId) - - - // routeManager := connect.NewRouteManager(client) - // contractManager := connect.NewContractManagerWithDefaults(client) - // client.Setup(routeManager, contractManager) - // go client.Run() - - auth := &connect.ClientAuth{ - ByJwt: jwt, - InstanceId: instanceId, - AppVersion: fmt.Sprintf("connectctl %s", ConnectCtlVersion), - } - for i := 0; i < transportCount; i += 1 { - platformTransport := connect.NewPlatformTransportWithDefaults( - cancelCtx, - fmt.Sprintf("%s/", connectUrl), - auth, - client.RouteManager(), - ) - defer platformTransport.Close() - // go platformTransport.Run(routeManager) - } - - - provideModes := map[protocol.ProvideMode]bool{ - protocol.ProvideMode_Network: true, - } - client.ContractManager().SetProvideModes(provideModes) - - - // FIXME break into 2k chunks? - acks := make(chan error) - go func() { - for i := 0; i < messageCount; i += 1 { - var content string - if 0 < messageCount { - content = fmt.Sprintf("[%d] %s", i, messageContent) - } else { - content = messageContent - } - message := &protocol.SimpleMessage{ - Content: content, - } - - client.Send( - connect.RequireToFrame(message), - destinationId, - func(err error) { - acks <- err - }, - ) - } - }() - for i := 0; i < messageCount; i += 1 { - select { - case err := <- acks: - if err == nil { - fmt.Printf("Message acked.\n") - } else { - fmt.Printf("Message not acked (%s).\n", err) - } - case <- time.After(timeout): - fmt.Printf("Message not acked (timeout).\n") - } - } + jwt, _ := opts.String("--jwt") + + var clientId connect.Id + + claims := gojwt.MapClaims{} + gojwt.NewParser().ParseUnverified(jwt, claims) + + jwtClientId, ok := claims["client_id"] + if !ok { + fmt.Printf("JWT does not have a client_id.\n") + return + } + switch v := jwtClientId.(type) { + case string: + var err error + clientId, err = connect.ParseId(v) + if err != nil { + fmt.Printf("JWT has invalid client_id (%s).\n", err) + return + } + default: + fmt.Printf("JWT has invalid client_id (%T).\n", v) + return + } + + fmt.Printf("client_id: %s\n", clientId.String()) + + connectUrl, err := opts.String("--connect_url") + if err != nil { + connectUrl = DefaultConnectUrl + } + + destinationIdStr, _ := opts.String("--destination_id") + destinationId, err := connect.ParseId(destinationIdStr) + if err != nil { + fmt.Printf("Invalid destination_id (%s).\n", err) + return + } + + instanceIdStr, err := opts.String("--instance_id") + var instanceId connect.Id + if err == nil { + instanceId, err = connect.ParseId(instanceIdStr) + if err != nil { + fmt.Printf("Invalid instance_id (%s).\n", err) + return + } + } else { + instanceId = connect.NewId() + } + + fmt.Printf("instance_id: %s\n", instanceId.String()) + + messageContent, _ := opts.String("") + + messageCount, err := opts.Int("--message_count") + if err != nil { + messageCount = 1 + } + + // need at least one. Use more for testing. + transportCount := 4 + + timeout := 30 * time.Second + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + client := connect.NewClientWithDefaults( + cancelCtx, + clientId, + ) + defer client.Close() + + // client.SetInstanceId(instanceId) + + // routeManager := connect.NewRouteManager(client) + // contractManager := connect.NewContractManagerWithDefaults(client) + // client.Setup(routeManager, contractManager) + // go client.Run() + + auth := &connect.ClientAuth{ + ByJwt: jwt, + InstanceId: instanceId, + AppVersion: fmt.Sprintf("connectctl %s", ConnectCtlVersion), + } + for i := 0; i < transportCount; i += 1 { + platformTransport := connect.NewPlatformTransportWithDefaults( + cancelCtx, + fmt.Sprintf("%s/", connectUrl), + auth, + client.RouteManager(), + ) + defer platformTransport.Close() + // go platformTransport.Run(routeManager) + } + + provideModes := map[protocol.ProvideMode]bool{ + protocol.ProvideMode_Network: true, + } + client.ContractManager().SetProvideModes(provideModes) + + // FIXME break into 2k chunks? + acks := make(chan error) + go func() { + for i := 0; i < messageCount; i += 1 { + var content string + if 0 < messageCount { + content = fmt.Sprintf("[%d] %s", i, messageContent) + } else { + content = messageContent + } + message := &protocol.SimpleMessage{ + Content: content, + } + + client.Send( + connect.RequireToFrame(message), + destinationId, + func(err error) { + acks <- err + }, + ) + } + }() + for i := 0; i < messageCount; i += 1 { + select { + case err := <-acks: + if err == nil { + fmt.Printf("Message acked.\n") + } else { + fmt.Printf("Message not acked (%s).\n", err) + } + case <-time.After(timeout): + fmt.Printf("Message not acked (timeout).\n") + } + } } - func sink(opts docopt.Opts) { - jwt, _ := opts.String("--jwt") - - var clientId connect.Id - - claims := gojwt.MapClaims{} - gojwt.NewParser().ParseUnverified(jwt, claims) - - jwtClientId, ok := claims["client_id"] - if !ok { - fmt.Printf("JWT does not have a client_id.\n") - return - } - switch v := jwtClientId.(type) { - case string: - var err error - clientId, err = connect.ParseId(v) - if err != nil { - fmt.Printf("JWT has invalid client_id (%s).\n", err) - return - } - default: - fmt.Printf("JWT has invalid client_id (%T).\n", v) - return - } - - fmt.Printf("client_id: %s\n", clientId.String()) - - - connectUrl, err := opts.String("--connect_url") - if err != nil { - connectUrl = DefaultConnectUrl - } - - messageCount, err := opts.Int("--message_count") - if err != nil { - messageCount = -1 - } - - transportCount := 4 - - instanceIdStr, err := opts.String("--instance_id") - var instanceId connect.Id - if err == nil { - instanceId, err = connect.ParseId(instanceIdStr) - if err != nil { - fmt.Printf("Invalid instance_id (%s).\n", err) - return - } - } else { - instanceId = connect.NewId() - } - - fmt.Printf("instance_id: %s\n", instanceId.String()) - - - cancelCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - - client := connect.NewClientWithDefaults( - cancelCtx, - clientId, - ) - defer client.Close() - - // client.SetInstanceId(instanceId) - - // routeManager := connect.NewRouteManager(client) - // contractManager := connect.NewContractManagerWithDefaults(client) - - // client.Setup(routeManager, contractManager) - // go client.Run() - - - provideModes := map[protocol.ProvideMode]bool{ - protocol.ProvideMode_Network: true, - } - client.ContractManager().SetProvideModes(provideModes) - - - auth := &connect.ClientAuth{ - ByJwt: jwt, - InstanceId: instanceId, - AppVersion: fmt.Sprintf("connectctl %s", ConnectCtlVersion), - } - for i := 0; i < transportCount; i += 1 { - platformTransport := connect.NewPlatformTransportWithDefaults( - cancelCtx, - fmt.Sprintf("%s/", connectUrl), - auth, - client.RouteManager(), - ) - defer platformTransport.Close() - // go platformTransport.Run(routeManager) - } - - type Receive struct { - sourceId connect.Id - frames []*protocol.Frame - provideMode protocol.ProvideMode - } - - receives := make(chan *Receive) - - client.AddReceiveCallback(func(sourceId connect.Id, frames []*protocol.Frame, provideMode protocol.ProvideMode) { - receives <- &Receive{ - sourceId: sourceId, - frames: frames, - provideMode: provideMode, - } - }) - - - // FIXME reassemble the chunks. Only a complete message counts as 1 against the message count - for i := 0; messageCount < 0 || i < messageCount; i += 1 { - select { - case receive := <- receives: - fmt.Printf("[%s %s] %s\n", receive.sourceId, receive.provideMode, receive.frames) - } - } + jwt, _ := opts.String("--jwt") + + var clientId connect.Id + + claims := gojwt.MapClaims{} + gojwt.NewParser().ParseUnverified(jwt, claims) + + jwtClientId, ok := claims["client_id"] + if !ok { + fmt.Printf("JWT does not have a client_id.\n") + return + } + switch v := jwtClientId.(type) { + case string: + var err error + clientId, err = connect.ParseId(v) + if err != nil { + fmt.Printf("JWT has invalid client_id (%s).\n", err) + return + } + default: + fmt.Printf("JWT has invalid client_id (%T).\n", v) + return + } + + fmt.Printf("client_id: %s\n", clientId.String()) + + connectUrl, err := opts.String("--connect_url") + if err != nil { + connectUrl = DefaultConnectUrl + } + + messageCount, err := opts.Int("--message_count") + if err != nil { + messageCount = -1 + } + + transportCount := 4 + + instanceIdStr, err := opts.String("--instance_id") + var instanceId connect.Id + if err == nil { + instanceId, err = connect.ParseId(instanceIdStr) + if err != nil { + fmt.Printf("Invalid instance_id (%s).\n", err) + return + } + } else { + instanceId = connect.NewId() + } + + fmt.Printf("instance_id: %s\n", instanceId.String()) + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + client := connect.NewClientWithDefaults( + cancelCtx, + clientId, + ) + defer client.Close() + + // client.SetInstanceId(instanceId) + + // routeManager := connect.NewRouteManager(client) + // contractManager := connect.NewContractManagerWithDefaults(client) + + // client.Setup(routeManager, contractManager) + // go client.Run() + + provideModes := map[protocol.ProvideMode]bool{ + protocol.ProvideMode_Network: true, + } + client.ContractManager().SetProvideModes(provideModes) + + auth := &connect.ClientAuth{ + ByJwt: jwt, + InstanceId: instanceId, + AppVersion: fmt.Sprintf("connectctl %s", ConnectCtlVersion), + } + for i := 0; i < transportCount; i += 1 { + platformTransport := connect.NewPlatformTransportWithDefaults( + cancelCtx, + fmt.Sprintf("%s/", connectUrl), + auth, + client.RouteManager(), + ) + defer platformTransport.Close() + // go platformTransport.Run(routeManager) + } + + type Receive struct { + sourceId connect.Id + frames []*protocol.Frame + provideMode protocol.ProvideMode + } + + receives := make(chan *Receive) + + client.AddReceiveCallback(func(sourceId connect.Id, frames []*protocol.Frame, provideMode protocol.ProvideMode) { + receives <- &Receive{ + sourceId: sourceId, + frames: frames, + provideMode: provideMode, + } + }) + + // FIXME reassemble the chunks. Only a complete message counts as 1 against the message count + for i := 0; messageCount < 0 || i < messageCount; i += 1 { + select { + case receive := <-receives: + fmt.Printf("[%s %s] %s\n", receive.sourceId, receive.provideMode, receive.frames) + } + } } diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..6d848e6 --- /dev/null +++ b/flake.lock @@ -0,0 +1,78 @@ +{ + "nodes": { + "devshell": { + "inputs": { + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1722113426, + "narHash": "sha256-Yo/3loq572A8Su6aY5GP56knpuKYRvM2a1meP9oJZCw=", + "owner": "numtide", + "repo": "devshell", + "rev": "67cce7359e4cd3c45296fb4aaf6a19e2a9c757ae", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "devshell", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1722073938, + "narHash": "sha256-OpX0StkL8vpXyWOGUD6G+MA26wAXK6SpT94kLJXo6B4=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e36e9f57337d0ff0cf77aceb58af4c805472bfae", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1724098845, + "narHash": "sha256-D5HwjQw/02fuXbR4LCTo64koglP2j99hkDR79/3yLOE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "f1bad50880bae73ff2d82fafc22010b4fc097a9c", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "devshell": "devshell", + "nixpkgs": "nixpkgs_2", + "systems": "systems" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..ad49573 --- /dev/null +++ b/flake.nix @@ -0,0 +1,45 @@ +{ + description = "ethereum-validator-watch"; + inputs = { + devshell = { + url = "github:numtide/devshell"; + inputs.systems.follows = "systems"; + }; + nixpkgs = { url = "github:NixOS/nixpkgs/nixos-24.05"; }; + + systems.url = "github:nix-systems/default"; + + }; + + outputs = { self, nixpkgs, systems, ... }@inputs: + let + eachSystem = f: + nixpkgs.lib.genAttrs (import systems) (system: + f (import nixpkgs { + inherit system; + config = { allowUnfree = true; }; + })); + + in { + devShells = eachSystem (pkgs: { + default = pkgs.mkShell { + shellHook = '' + # Set here the env vars you want to be available in the shell + ''; + hardeningDisable = [ "all" ]; + + packages = [ + pkgs.go_1_22 + pkgs.goda + pkgs.gopls + pkgs.gops + pkgs.go-tools + pkgs.tmux + pkgs.protobuf + pkgs.protoc-gen-go + ]; + }; + }); + }; +} + diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1249a6d --- /dev/null +++ b/go.mod @@ -0,0 +1,18 @@ +module github.com/bringyour/connect + +go 1.21.5 + +require ( + github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 + github.com/go-playground/assert/v2 v2.2.0 + github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/golang/glog v1.2.2 + github.com/google/gopacket v1.1.19 + github.com/gorilla/websocket v1.5.3 + github.com/oklog/ulid/v2 v2.1.0 + golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa + golang.org/x/term v0.23.0 + google.golang.org/protobuf v1.34.2 +) + +require golang.org/x/sys v0.23.0 // indirect diff --git a/provider/go.sum b/go.sum similarity index 64% rename from provider/go.sum rename to go.sum index 703adb8..b078f85 100644 --- a/provider/go.sum +++ b/go.sum @@ -2,23 +2,23 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= -github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= -github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= +github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU= github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -26,12 +26,12 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/protocol/Makefile b/protocol/Makefile deleted file mode 100644 index 85a5ec1..0000000 --- a/protocol/Makefile +++ /dev/null @@ -1,17 +0,0 @@ - -all: clean build - -clean: - rm -rf build - -build: - mkdir build - protoc -I=. --go_out=build *.proto - cp go.mod build/bringyour.com/protocol/ -# cd build/bringyour.com/protocol; go mod init bringyour.com/protocol - cd build/bringyour.com/protocol; go mod tidy - -install_protoc: - # install protoc from https://github.com/protocolbuffers/protobuf/releases - # xattr -d com.apple.quarantine `which protoc` - go install google.golang.org/protobuf/cmd/protoc-gen-go@latest diff --git a/protocol/frame.pb.go b/protocol/frame.pb.go new file mode 100644 index 0000000..c6283a5 --- /dev/null +++ b/protocol/frame.pb.go @@ -0,0 +1,350 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.34.1 +// protoc v4.24.4 +// source: frame.proto + +package protocol + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// frames are used at all levels +// flatten all message types into this enum +type MessageType int32 + +const ( + MessageType_TransferPack MessageType = 0 + MessageType_TransferAck MessageType = 1 + MessageType_TransferContract MessageType = 2 + MessageType_TransferProvide MessageType = 3 + MessageType_TransferAuth MessageType = 4 + MessageType_TransferCreateStream MessageType = 5 + MessageType_TransferCreateStreamResult MessageType = 6 + // REMOVED + MessageType_TransferCloseStream MessageType = 7 + MessageType_TransferStreamOpen MessageType = 8 + MessageType_TransferStreamClose MessageType = 9 + MessageType_TransferCreateContract MessageType = 10 + MessageType_TransferCreateContractResult MessageType = 11 + MessageType_TransferCloseContract MessageType = 12 + MessageType_TransferPeerAudit MessageType = 13 + MessageType_TestSimpleMessage MessageType = 14 + MessageType_IpIpPacketToProvider MessageType = 15 + MessageType_IpIpPacketFromProvider MessageType = 16 + MessageType_IpIpPing MessageType = 17 +) + +// Enum value maps for MessageType. +var ( + MessageType_name = map[int32]string{ + 0: "TransferPack", + 1: "TransferAck", + 2: "TransferContract", + 3: "TransferProvide", + 4: "TransferAuth", + 5: "TransferCreateStream", + 6: "TransferCreateStreamResult", + 7: "TransferCloseStream", + 8: "TransferStreamOpen", + 9: "TransferStreamClose", + 10: "TransferCreateContract", + 11: "TransferCreateContractResult", + 12: "TransferCloseContract", + 13: "TransferPeerAudit", + 14: "TestSimpleMessage", + 15: "IpIpPacketToProvider", + 16: "IpIpPacketFromProvider", + 17: "IpIpPing", + } + MessageType_value = map[string]int32{ + "TransferPack": 0, + "TransferAck": 1, + "TransferContract": 2, + "TransferProvide": 3, + "TransferAuth": 4, + "TransferCreateStream": 5, + "TransferCreateStreamResult": 6, + "TransferCloseStream": 7, + "TransferStreamOpen": 8, + "TransferStreamClose": 9, + "TransferCreateContract": 10, + "TransferCreateContractResult": 11, + "TransferCloseContract": 12, + "TransferPeerAudit": 13, + "TestSimpleMessage": 14, + "IpIpPacketToProvider": 15, + "IpIpPacketFromProvider": 16, + "IpIpPing": 17, + } +) + +func (x MessageType) Enum() *MessageType { + p := new(MessageType) + *p = x + return p +} + +func (x MessageType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MessageType) Descriptor() protoreflect.EnumDescriptor { + return file_frame_proto_enumTypes[0].Descriptor() +} + +func (MessageType) Type() protoreflect.EnumType { + return &file_frame_proto_enumTypes[0] +} + +func (x MessageType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MessageType.Descriptor instead. +func (MessageType) EnumDescriptor() ([]byte, []int) { + return file_frame_proto_rawDescGZIP(), []int{0} +} + +type Frame struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + MessageType MessageType `protobuf:"varint,1,opt,name=message_type,json=messageType,proto3,enum=bringyour.MessageType" json:"message_type,omitempty"` + MessageBytes []byte `protobuf:"bytes,2,opt,name=message_bytes,json=messageBytes,proto3" json:"message_bytes,omitempty"` +} + +func (x *Frame) Reset() { + *x = Frame{} + if protoimpl.UnsafeEnabled { + mi := &file_frame_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Frame) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Frame) ProtoMessage() {} + +func (x *Frame) ProtoReflect() protoreflect.Message { + mi := &file_frame_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Frame.ProtoReflect.Descriptor instead. +func (*Frame) Descriptor() ([]byte, []int) { + return file_frame_proto_rawDescGZIP(), []int{0} +} + +func (x *Frame) GetMessageType() MessageType { + if x != nil { + return x.MessageType + } + return MessageType_TransferPack +} + +func (x *Frame) GetMessageBytes() []byte { + if x != nil { + return x.MessageBytes + } + return nil +} + +type FilteredFrame struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + MessageType MessageType `protobuf:"varint,1,opt,name=message_type,json=messageType,proto3,enum=bringyour.MessageType" json:"message_type,omitempty"` +} + +func (x *FilteredFrame) Reset() { + *x = FilteredFrame{} + if protoimpl.UnsafeEnabled { + mi := &file_frame_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FilteredFrame) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FilteredFrame) ProtoMessage() {} + +func (x *FilteredFrame) ProtoReflect() protoreflect.Message { + mi := &file_frame_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FilteredFrame.ProtoReflect.Descriptor instead. +func (*FilteredFrame) Descriptor() ([]byte, []int) { + return file_frame_proto_rawDescGZIP(), []int{1} +} + +func (x *FilteredFrame) GetMessageType() MessageType { + if x != nil { + return x.MessageType + } + return MessageType_TransferPack +} + +var File_frame_proto protoreflect.FileDescriptor + +var file_frame_proto_rawDesc = []byte{ + 0x0a, 0x0b, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x62, + 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, 0x72, 0x22, 0x67, 0x0a, 0x05, 0x46, 0x72, 0x61, 0x6d, + 0x65, 0x12, 0x39, 0x0a, 0x0c, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, + 0x6f, 0x75, 0x72, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, + 0x0b, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x0d, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x79, 0x74, 0x65, + 0x73, 0x22, 0x4a, 0x0a, 0x0d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x65, 0x64, 0x46, 0x72, 0x61, + 0x6d, 0x65, 0x12, 0x39, 0x0a, 0x0c, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, + 0x79, 0x6f, 0x75, 0x72, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, + 0x52, 0x0b, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x2a, 0xbc, 0x03, + 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, + 0x0c, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x50, 0x61, 0x63, 0x6b, 0x10, 0x00, 0x12, + 0x0f, 0x0a, 0x0b, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x41, 0x63, 0x6b, 0x10, 0x01, + 0x12, 0x14, 0x0a, 0x10, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x74, + 0x72, 0x61, 0x63, 0x74, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, + 0x65, 0x72, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x10, 0x03, 0x12, 0x10, 0x0a, 0x0c, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x41, 0x75, 0x74, 0x68, 0x10, 0x04, 0x12, 0x18, 0x0a, + 0x14, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x10, 0x05, 0x12, 0x1e, 0x0a, 0x1a, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x65, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x10, 0x06, 0x12, 0x17, 0x0a, 0x13, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x65, 0x72, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x10, 0x07, + 0x12, 0x16, 0x0a, 0x12, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x4f, 0x70, 0x65, 0x6e, 0x10, 0x08, 0x12, 0x17, 0x0a, 0x13, 0x54, 0x72, 0x61, 0x6e, + 0x73, 0x66, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x10, + 0x09, 0x12, 0x1a, 0x0a, 0x16, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x10, 0x0a, 0x12, 0x20, 0x0a, + 0x1c, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, + 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x10, 0x0b, 0x12, + 0x19, 0x0a, 0x15, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x43, 0x6c, 0x6f, 0x73, 0x65, + 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x10, 0x0c, 0x12, 0x15, 0x0a, 0x11, 0x54, 0x72, + 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x50, 0x65, 0x65, 0x72, 0x41, 0x75, 0x64, 0x69, 0x74, 0x10, + 0x0d, 0x12, 0x15, 0x0a, 0x11, 0x54, 0x65, 0x73, 0x74, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x10, 0x0e, 0x12, 0x18, 0x0a, 0x14, 0x49, 0x70, 0x49, 0x70, + 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x6f, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x10, 0x0f, 0x12, 0x1a, 0x0a, 0x16, 0x49, 0x70, 0x49, 0x70, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, + 0x46, 0x72, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x10, 0x10, 0x12, 0x0c, + 0x0a, 0x08, 0x49, 0x70, 0x49, 0x70, 0x50, 0x69, 0x6e, 0x67, 0x10, 0x11, 0x42, 0x27, 0x5a, 0x25, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x62, 0x72, 0x69, 0x6e, 0x67, + 0x79, 0x6f, 0x75, 0x72, 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_frame_proto_rawDescOnce sync.Once + file_frame_proto_rawDescData = file_frame_proto_rawDesc +) + +func file_frame_proto_rawDescGZIP() []byte { + file_frame_proto_rawDescOnce.Do(func() { + file_frame_proto_rawDescData = protoimpl.X.CompressGZIP(file_frame_proto_rawDescData) + }) + return file_frame_proto_rawDescData +} + +var file_frame_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_frame_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_frame_proto_goTypes = []interface{}{ + (MessageType)(0), // 0: bringyour.MessageType + (*Frame)(nil), // 1: bringyour.Frame + (*FilteredFrame)(nil), // 2: bringyour.FilteredFrame +} +var file_frame_proto_depIdxs = []int32{ + 0, // 0: bringyour.Frame.message_type:type_name -> bringyour.MessageType + 0, // 1: bringyour.FilteredFrame.message_type:type_name -> bringyour.MessageType + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_frame_proto_init() } +func file_frame_proto_init() { + if File_frame_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_frame_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Frame); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_frame_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FilteredFrame); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_frame_proto_rawDesc, + NumEnums: 1, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_frame_proto_goTypes, + DependencyIndexes: file_frame_proto_depIdxs, + EnumInfos: file_frame_proto_enumTypes, + MessageInfos: file_frame_proto_msgTypes, + }.Build() + File_frame_proto = out.File + file_frame_proto_rawDesc = nil + file_frame_proto_goTypes = nil + file_frame_proto_depIdxs = nil +} diff --git a/protocol/frame.proto b/protocol/frame.proto index 5c59161..9999483 100644 --- a/protocol/frame.proto +++ b/protocol/frame.proto @@ -1,7 +1,7 @@ syntax = "proto3"; package bringyour; -option go_package = "bringyour.com/protocol"; +option go_package = "github.com/bringyour/connect/protocol"; // frames are used at all levels diff --git a/protocol/generate.go b/protocol/generate.go new file mode 100644 index 0000000..b78e8da --- /dev/null +++ b/protocol/generate.go @@ -0,0 +1,3 @@ +package protocol + +//go:generate protoc -I=. --go_out=. --go_opt=paths=source_relative transfer.proto frame.proto ip.proto test.proto diff --git a/protocol/go.mod b/protocol/go.mod deleted file mode 100644 index b57ff3f..0000000 --- a/protocol/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module bringyour.com/protocol - -go 1.22.0 - -require google.golang.org/protobuf v1.33.0 diff --git a/protocol/ip.pb.go b/protocol/ip.pb.go new file mode 100644 index 0000000..a90da76 --- /dev/null +++ b/protocol/ip.pb.go @@ -0,0 +1,329 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.34.1 +// protoc v4.24.4 +// source: ip.proto + +package protocol + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// a raw packet +// the provider is expected to do its own parsing to confirm the data verbatim +type IpPacket struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + PacketBytes []byte `protobuf:"bytes,1,opt,name=packet_bytes,json=packetBytes,proto3" json:"packet_bytes,omitempty"` +} + +func (x *IpPacket) Reset() { + *x = IpPacket{} + if protoimpl.UnsafeEnabled { + mi := &file_ip_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IpPacket) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IpPacket) ProtoMessage() {} + +func (x *IpPacket) ProtoReflect() protoreflect.Message { + mi := &file_ip_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IpPacket.ProtoReflect.Descriptor instead. +func (*IpPacket) Descriptor() ([]byte, []int) { + return file_ip_proto_rawDescGZIP(), []int{0} +} + +func (x *IpPacket) GetPacketBytes() []byte { + if x != nil { + return x.PacketBytes + } + return nil +} + +type IpPacketToProvider struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + IpPacket *IpPacket `protobuf:"bytes,1,opt,name=ip_packet,json=ipPacket,proto3" json:"ip_packet,omitempty"` +} + +func (x *IpPacketToProvider) Reset() { + *x = IpPacketToProvider{} + if protoimpl.UnsafeEnabled { + mi := &file_ip_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IpPacketToProvider) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IpPacketToProvider) ProtoMessage() {} + +func (x *IpPacketToProvider) ProtoReflect() protoreflect.Message { + mi := &file_ip_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IpPacketToProvider.ProtoReflect.Descriptor instead. +func (*IpPacketToProvider) Descriptor() ([]byte, []int) { + return file_ip_proto_rawDescGZIP(), []int{1} +} + +func (x *IpPacketToProvider) GetIpPacket() *IpPacket { + if x != nil { + return x.IpPacket + } + return nil +} + +type IpPacketFromProvider struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + IpPacket *IpPacket `protobuf:"bytes,1,opt,name=ip_packet,json=ipPacket,proto3" json:"ip_packet,omitempty"` +} + +func (x *IpPacketFromProvider) Reset() { + *x = IpPacketFromProvider{} + if protoimpl.UnsafeEnabled { + mi := &file_ip_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IpPacketFromProvider) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IpPacketFromProvider) ProtoMessage() {} + +func (x *IpPacketFromProvider) ProtoReflect() protoreflect.Message { + mi := &file_ip_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IpPacketFromProvider.ProtoReflect.Descriptor instead. +func (*IpPacketFromProvider) Descriptor() ([]byte, []int) { + return file_ip_proto_rawDescGZIP(), []int{2} +} + +func (x *IpPacketFromProvider) GetIpPacket() *IpPacket { + if x != nil { + return x.IpPacket + } + return nil +} + +type IpPing struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *IpPing) Reset() { + *x = IpPing{} + if protoimpl.UnsafeEnabled { + mi := &file_ip_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IpPing) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IpPing) ProtoMessage() {} + +func (x *IpPing) ProtoReflect() protoreflect.Message { + mi := &file_ip_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IpPing.ProtoReflect.Descriptor instead. +func (*IpPing) Descriptor() ([]byte, []int) { + return file_ip_proto_rawDescGZIP(), []int{3} +} + +var File_ip_proto protoreflect.FileDescriptor + +var file_ip_proto_rawDesc = []byte{ + 0x0a, 0x08, 0x69, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x62, 0x72, 0x69, 0x6e, + 0x67, 0x79, 0x6f, 0x75, 0x72, 0x22, 0x2d, 0x0a, 0x08, 0x49, 0x70, 0x50, 0x61, 0x63, 0x6b, 0x65, + 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x62, 0x79, 0x74, 0x65, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x42, + 0x79, 0x74, 0x65, 0x73, 0x22, 0x46, 0x0a, 0x12, 0x49, 0x70, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, + 0x54, 0x6f, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x30, 0x0a, 0x09, 0x69, 0x70, + 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, + 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, 0x72, 0x2e, 0x49, 0x70, 0x50, 0x61, 0x63, 0x6b, + 0x65, 0x74, 0x52, 0x08, 0x69, 0x70, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x22, 0x48, 0x0a, 0x14, + 0x49, 0x70, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x76, + 0x69, 0x64, 0x65, 0x72, 0x12, 0x30, 0x0a, 0x09, 0x69, 0x70, 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x65, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, + 0x6f, 0x75, 0x72, 0x2e, 0x49, 0x70, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x08, 0x69, 0x70, + 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x22, 0x08, 0x0a, 0x06, 0x49, 0x70, 0x50, 0x69, 0x6e, 0x67, + 0x42, 0x27, 0x5a, 0x25, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x62, + 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, 0x72, 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, +} + +var ( + file_ip_proto_rawDescOnce sync.Once + file_ip_proto_rawDescData = file_ip_proto_rawDesc +) + +func file_ip_proto_rawDescGZIP() []byte { + file_ip_proto_rawDescOnce.Do(func() { + file_ip_proto_rawDescData = protoimpl.X.CompressGZIP(file_ip_proto_rawDescData) + }) + return file_ip_proto_rawDescData +} + +var file_ip_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_ip_proto_goTypes = []interface{}{ + (*IpPacket)(nil), // 0: bringyour.IpPacket + (*IpPacketToProvider)(nil), // 1: bringyour.IpPacketToProvider + (*IpPacketFromProvider)(nil), // 2: bringyour.IpPacketFromProvider + (*IpPing)(nil), // 3: bringyour.IpPing +} +var file_ip_proto_depIdxs = []int32{ + 0, // 0: bringyour.IpPacketToProvider.ip_packet:type_name -> bringyour.IpPacket + 0, // 1: bringyour.IpPacketFromProvider.ip_packet:type_name -> bringyour.IpPacket + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_ip_proto_init() } +func file_ip_proto_init() { + if File_ip_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_ip_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IpPacket); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ip_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IpPacketToProvider); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ip_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IpPacketFromProvider); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ip_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IpPing); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_ip_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_ip_proto_goTypes, + DependencyIndexes: file_ip_proto_depIdxs, + MessageInfos: file_ip_proto_msgTypes, + }.Build() + File_ip_proto = out.File + file_ip_proto_rawDesc = nil + file_ip_proto_goTypes = nil + file_ip_proto_depIdxs = nil +} diff --git a/protocol/ip.proto b/protocol/ip.proto index 027e5ee..b920156 100644 --- a/protocol/ip.proto +++ b/protocol/ip.proto @@ -1,7 +1,7 @@ syntax = "proto3"; package bringyour; -option go_package = "bringyour.com/protocol"; +option go_package = "github.com/bringyour/connect/protocol"; // a raw packet diff --git a/protocol/test.pb.go b/protocol/test.pb.go new file mode 100644 index 0000000..8a7b937 --- /dev/null +++ b/protocol/test.pb.go @@ -0,0 +1,164 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.34.1 +// protoc v4.24.4 +// source: test.proto + +package protocol + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SimpleMessage struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + MessageIndex uint32 `protobuf:"varint,1,opt,name=message_index,json=messageIndex,proto3" json:"message_index,omitempty"` + MessageCount uint32 `protobuf:"varint,2,opt,name=message_count,json=messageCount,proto3" json:"message_count,omitempty"` + Content string `protobuf:"bytes,3,opt,name=content,proto3" json:"content,omitempty"` +} + +func (x *SimpleMessage) Reset() { + *x = SimpleMessage{} + if protoimpl.UnsafeEnabled { + mi := &file_test_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SimpleMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SimpleMessage) ProtoMessage() {} + +func (x *SimpleMessage) ProtoReflect() protoreflect.Message { + mi := &file_test_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SimpleMessage.ProtoReflect.Descriptor instead. +func (*SimpleMessage) Descriptor() ([]byte, []int) { + return file_test_proto_rawDescGZIP(), []int{0} +} + +func (x *SimpleMessage) GetMessageIndex() uint32 { + if x != nil { + return x.MessageIndex + } + return 0 +} + +func (x *SimpleMessage) GetMessageCount() uint32 { + if x != nil { + return x.MessageCount + } + return 0 +} + +func (x *SimpleMessage) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +var File_test_proto protoreflect.FileDescriptor + +var file_test_proto_rawDesc = []byte{ + 0x0a, 0x0a, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x62, 0x72, + 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, 0x72, 0x22, 0x73, 0x0a, 0x0d, 0x53, 0x69, 0x6d, 0x70, 0x6c, + 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x0c, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x23, 0x0a, + 0x0d, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, + 0x6e, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x42, 0x27, 0x5a, 0x25, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x62, 0x72, 0x69, 0x6e, 0x67, + 0x79, 0x6f, 0x75, 0x72, 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_test_proto_rawDescOnce sync.Once + file_test_proto_rawDescData = file_test_proto_rawDesc +) + +func file_test_proto_rawDescGZIP() []byte { + file_test_proto_rawDescOnce.Do(func() { + file_test_proto_rawDescData = protoimpl.X.CompressGZIP(file_test_proto_rawDescData) + }) + return file_test_proto_rawDescData +} + +var file_test_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_test_proto_goTypes = []interface{}{ + (*SimpleMessage)(nil), // 0: bringyour.SimpleMessage +} +var file_test_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_test_proto_init() } +func file_test_proto_init() { + if File_test_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_test_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SimpleMessage); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_test_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_test_proto_goTypes, + DependencyIndexes: file_test_proto_depIdxs, + MessageInfos: file_test_proto_msgTypes, + }.Build() + File_test_proto = out.File + file_test_proto_rawDesc = nil + file_test_proto_goTypes = nil + file_test_proto_depIdxs = nil +} diff --git a/protocol/test.proto b/protocol/test.proto index 125d481..ccbefae 100644 --- a/protocol/test.proto +++ b/protocol/test.proto @@ -1,7 +1,7 @@ syntax = "proto3"; package bringyour; -option go_package = "bringyour.com/protocol"; +option go_package = "github.com/bringyour/connect/protocol"; message SimpleMessage { diff --git a/protocol/transfer.pb.go b/protocol/transfer.pb.go new file mode 100644 index 0000000..b9cbd57 --- /dev/null +++ b/protocol/transfer.pb.go @@ -0,0 +1,1949 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.34.1 +// protoc v4.24.4 +// source: transfer.proto + +package protocol + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// CHANGE FROM PRIOR VERSION, modes should be considered individually as a mask and do not imply other modes +type ProvideMode int32 + +const ( + ProvideMode_None ProvideMode = 0 // this is interpreted the same as a `null` provide mode + ProvideMode_Network ProvideMode = 1 + ProvideMode_FriendsAndFamily ProvideMode = 2 + ProvideMode_Public ProvideMode = 3 + ProvideMode_Stream ProvideMode = 4 +) + +// Enum value maps for ProvideMode. +var ( + ProvideMode_name = map[int32]string{ + 0: "None", + 1: "Network", + 2: "FriendsAndFamily", + 3: "Public", + 4: "Stream", + } + ProvideMode_value = map[string]int32{ + "None": 0, + "Network": 1, + "FriendsAndFamily": 2, + "Public": 3, + "Stream": 4, + } +) + +func (x ProvideMode) Enum() *ProvideMode { + p := new(ProvideMode) + *p = x + return p +} + +func (x ProvideMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ProvideMode) Descriptor() protoreflect.EnumDescriptor { + return file_transfer_proto_enumTypes[0].Descriptor() +} + +func (ProvideMode) Type() protoreflect.EnumType { + return &file_transfer_proto_enumTypes[0] +} + +func (x ProvideMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ProvideMode.Descriptor instead. +func (ProvideMode) EnumDescriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{0} +} + +type ContractError int32 + +const ( + ContractError_NoPermission ContractError = 0 + ContractError_InsufficientBalance ContractError = 1 + ContractError_Setup ContractError = 2 +) + +// Enum value maps for ContractError. +var ( + ContractError_name = map[int32]string{ + 0: "NoPermission", + 1: "InsufficientBalance", + 2: "Setup", + } + ContractError_value = map[string]int32{ + "NoPermission": 0, + "InsufficientBalance": 1, + "Setup": 2, + } +) + +func (x ContractError) Enum() *ContractError { + p := new(ContractError) + *p = x + return p +} + +func (x ContractError) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ContractError) Descriptor() protoreflect.EnumDescriptor { + return file_transfer_proto_enumTypes[1].Descriptor() +} + +func (ContractError) Type() protoreflect.EnumType { + return &file_transfer_proto_enumTypes[1] +} + +func (x ContractError) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ContractError.Descriptor instead. +func (ContractError) EnumDescriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{1} +} + +// stream_id replaces the source_id and destination_id +type TransferPath struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ulid + SourceId []byte `protobuf:"bytes,2,opt,name=source_id,json=sourceId,proto3,oneof" json:"source_id,omitempty"` + // ulid + DestinationId []byte `protobuf:"bytes,1,opt,name=destination_id,json=destinationId,proto3,oneof" json:"destination_id,omitempty"` + // ulid + StreamId []byte `protobuf:"bytes,3,opt,name=stream_id,json=streamId,proto3,oneof" json:"stream_id,omitempty"` +} + +func (x *TransferPath) Reset() { + *x = TransferPath{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TransferPath) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TransferPath) ProtoMessage() {} + +func (x *TransferPath) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TransferPath.ProtoReflect.Descriptor instead. +func (*TransferPath) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{0} +} + +func (x *TransferPath) GetSourceId() []byte { + if x != nil { + return x.SourceId + } + return nil +} + +func (x *TransferPath) GetDestinationId() []byte { + if x != nil { + return x.DestinationId + } + return nil +} + +func (x *TransferPath) GetStreamId() []byte { + if x != nil { + return x.StreamId + } + return nil +} + +type TransferFrame struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TransferPath *TransferPath `protobuf:"bytes,1,opt,name=transfer_path,json=transferPath,proto3" json:"transfer_path,omitempty"` + Frame *Frame `protobuf:"bytes,2,opt,name=frame,proto3" json:"frame,omitempty"` +} + +func (x *TransferFrame) Reset() { + *x = TransferFrame{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TransferFrame) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TransferFrame) ProtoMessage() {} + +func (x *TransferFrame) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TransferFrame.ProtoReflect.Descriptor instead. +func (*TransferFrame) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{1} +} + +func (x *TransferFrame) GetTransferPath() *TransferPath { + if x != nil { + return x.TransferPath + } + return nil +} + +func (x *TransferFrame) GetFrame() *Frame { + if x != nil { + return x.Frame + } + return nil +} + +// this is the minimal subset of `TransferFrame` used when making a routing decision +type FilteredTransferFrame struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TransferPath *TransferPath `protobuf:"bytes,1,opt,name=transfer_path,json=transferPath,proto3" json:"transfer_path,omitempty"` +} + +func (x *FilteredTransferFrame) Reset() { + *x = FilteredTransferFrame{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FilteredTransferFrame) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FilteredTransferFrame) ProtoMessage() {} + +func (x *FilteredTransferFrame) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FilteredTransferFrame.ProtoReflect.Descriptor instead. +func (*FilteredTransferFrame) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{2} +} + +func (x *FilteredTransferFrame) GetTransferPath() *TransferPath { + if x != nil { + return x.TransferPath + } + return nil +} + +type FilteredTransferFrameWithFrame struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TransferPath *TransferPath `protobuf:"bytes,1,opt,name=transfer_path,json=transferPath,proto3" json:"transfer_path,omitempty"` + Frame *FilteredFrame `protobuf:"bytes,2,opt,name=frame,proto3" json:"frame,omitempty"` +} + +func (x *FilteredTransferFrameWithFrame) Reset() { + *x = FilteredTransferFrameWithFrame{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FilteredTransferFrameWithFrame) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FilteredTransferFrameWithFrame) ProtoMessage() {} + +func (x *FilteredTransferFrameWithFrame) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FilteredTransferFrameWithFrame.ProtoReflect.Descriptor instead. +func (*FilteredTransferFrameWithFrame) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{3} +} + +func (x *FilteredTransferFrameWithFrame) GetTransferPath() *TransferPath { + if x != nil { + return x.TransferPath + } + return nil +} + +func (x *FilteredTransferFrameWithFrame) GetFrame() *FilteredFrame { + if x != nil { + return x.Frame + } + return nil +} + +type Pack struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ulid + MessageId []byte `protobuf:"bytes,1,opt,name=message_id,json=messageId,proto3" json:"message_id,omitempty"` + // ulid + SequenceId []byte `protobuf:"bytes,2,opt,name=sequence_id,json=sequenceId,proto3" json:"sequence_id,omitempty"` + SequenceNumber uint64 `protobuf:"varint,3,opt,name=sequence_number,json=sequenceNumber,proto3" json:"sequence_number,omitempty"` + // this marks the head message in the sender's sequence + // it means that any message with an earlier sequence_id won't arrive + // and is used when a receiver is recreated with zero state + Head bool `protobuf:"varint,4,opt,name=head,proto3" json:"head,omitempty"` + Frames []*Frame `protobuf:"bytes,5,rep,name=frames,proto3" json:"frames,omitempty"` + // when true, deliver out of sequence with no acks and no retry + // use true when there is an external transfer control + // default to false for backward compatibility + Nack bool `protobuf:"varint,6,opt,name=nack,proto3" json:"nack,omitempty"` + ContractFrame *Frame `protobuf:"bytes,7,opt,name=contract_frame,json=contractFrame,proto3,oneof" json:"contract_frame,omitempty"` +} + +func (x *Pack) Reset() { + *x = Pack{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Pack) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Pack) ProtoMessage() {} + +func (x *Pack) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Pack.ProtoReflect.Descriptor instead. +func (*Pack) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{4} +} + +func (x *Pack) GetMessageId() []byte { + if x != nil { + return x.MessageId + } + return nil +} + +func (x *Pack) GetSequenceId() []byte { + if x != nil { + return x.SequenceId + } + return nil +} + +func (x *Pack) GetSequenceNumber() uint64 { + if x != nil { + return x.SequenceNumber + } + return 0 +} + +func (x *Pack) GetHead() bool { + if x != nil { + return x.Head + } + return false +} + +func (x *Pack) GetFrames() []*Frame { + if x != nil { + return x.Frames + } + return nil +} + +func (x *Pack) GetNack() bool { + if x != nil { + return x.Nack + } + return false +} + +func (x *Pack) GetContractFrame() *Frame { + if x != nil { + return x.ContractFrame + } + return nil +} + +// used for deep message inspection +type FilteredPack struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContractFrame *Frame `protobuf:"bytes,7,opt,name=contract_frame,json=contractFrame,proto3,oneof" json:"contract_frame,omitempty"` +} + +func (x *FilteredPack) Reset() { + *x = FilteredPack{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FilteredPack) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FilteredPack) ProtoMessage() {} + +func (x *FilteredPack) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FilteredPack.ProtoReflect.Descriptor instead. +func (*FilteredPack) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{5} +} + +func (x *FilteredPack) GetContractFrame() *Frame { + if x != nil { + return x.ContractFrame + } + return nil +} + +// ack of a later message means all prior messages have been ackd +type Ack struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ulid + MessageId []byte `protobuf:"bytes,1,opt,name=message_id,json=messageId,proto3" json:"message_id,omitempty"` + // ulid + SequenceId []byte `protobuf:"bytes,2,opt,name=sequence_id,json=sequenceId,proto3" json:"sequence_id,omitempty"` + // all data buffered in the receiver is acked with `selective=true`. When released it is acked with `selective=false` + Selective bool `protobuf:"varint,3,opt,name=selective,proto3" json:"selective,omitempty"` +} + +func (x *Ack) Reset() { + *x = Ack{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Ack) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Ack) ProtoMessage() {} + +func (x *Ack) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Ack.ProtoReflect.Descriptor instead. +func (*Ack) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{6} +} + +func (x *Ack) GetMessageId() []byte { + if x != nil { + return x.MessageId + } + return nil +} + +func (x *Ack) GetSequenceId() []byte { + if x != nil { + return x.SequenceId + } + return nil +} + +func (x *Ack) GetSelective() bool { + if x != nil { + return x.Selective + } + return false +} + +// this is a connection local message that is sent at the start of each connection to the control platform +// there is no response. the other side will close the connection if no auth +// no auth can happen if the jwt is not valid, or the client_id is not associated with the jwt +// note that association of the client_id to the jwt uses the api (see openapi spec in the api dir) +type Auth struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // the jwt will have the client_id in it + ByJwt string `protobuf:"bytes,1,opt,name=by_jwt,json=byJwt,proto3" json:"by_jwt,omitempty"` + // the version of the app, a semver managed by warp + AppVersion string `protobuf:"bytes,2,opt,name=app_version,json=appVersion,proto3" json:"app_version,omitempty"` + // the pair (client_id, instance_id) represents a single in memory instance + // this helps the platform distinguish multiple instances from multiple transports of the same instance + InstanceId []byte `protobuf:"bytes,3,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` +} + +func (x *Auth) Reset() { + *x = Auth{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Auth) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Auth) ProtoMessage() {} + +func (x *Auth) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Auth.ProtoReflect.Descriptor instead. +func (*Auth) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{7} +} + +func (x *Auth) GetByJwt() string { + if x != nil { + return x.ByJwt + } + return "" +} + +func (x *Auth) GetAppVersion() string { + if x != nil { + return x.AppVersion + } + return "" +} + +func (x *Auth) GetInstanceId() []byte { + if x != nil { + return x.InstanceId + } + return nil +} + +type Provide struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Keys []*ProvideKey `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` +} + +func (x *Provide) Reset() { + *x = Provide{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Provide) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Provide) ProtoMessage() {} + +func (x *Provide) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Provide.ProtoReflect.Descriptor instead. +func (*Provide) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{8} +} + +func (x *Provide) GetKeys() []*ProvideKey { + if x != nil { + return x.Keys + } + return nil +} + +type ProvideKey struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Mode ProvideMode `protobuf:"varint,1,opt,name=mode,proto3,enum=bringyour.ProvideMode" json:"mode,omitempty"` + // used to sign the `StoredContract` bytes + ProvideSecretKey []byte `protobuf:"bytes,2,opt,name=provide_secret_key,json=provideSecretKey,proto3" json:"provide_secret_key,omitempty"` +} + +func (x *ProvideKey) Reset() { + *x = ProvideKey{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProvideKey) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProvideKey) ProtoMessage() {} + +func (x *ProvideKey) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProvideKey.ProtoReflect.Descriptor instead. +func (*ProvideKey) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{9} +} + +func (x *ProvideKey) GetMode() ProvideMode { + if x != nil { + return x.Mode + } + return ProvideMode_None +} + +func (x *ProvideKey) GetProvideSecretKey() []byte { + if x != nil { + return x.ProvideSecretKey + } + return nil +} + +// each hop on the stream receives this to configure its state, +// including the first and last hops +// this is sent each time a stream contract is created +type StreamOpen struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ulid + SourceId []byte `protobuf:"bytes,2,opt,name=source_id,json=sourceId,proto3,oneof" json:"source_id,omitempty"` + // ulid + DestinationId []byte `protobuf:"bytes,1,opt,name=destination_id,json=destinationId,proto3,oneof" json:"destination_id,omitempty"` + // ulid + StreamId []byte `protobuf:"bytes,3,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` +} + +func (x *StreamOpen) Reset() { + *x = StreamOpen{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamOpen) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamOpen) ProtoMessage() {} + +func (x *StreamOpen) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamOpen.ProtoReflect.Descriptor instead. +func (*StreamOpen) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{10} +} + +func (x *StreamOpen) GetSourceId() []byte { + if x != nil { + return x.SourceId + } + return nil +} + +func (x *StreamOpen) GetDestinationId() []byte { + if x != nil { + return x.DestinationId + } + return nil +} + +func (x *StreamOpen) GetStreamId() []byte { + if x != nil { + return x.StreamId + } + return nil +} + +// each hop on the stream receives this to configure its state +// this is sent when all open contracts for the stream are closed +type StreamClose struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ulid + StreamId []byte `protobuf:"bytes,3,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` +} + +func (x *StreamClose) Reset() { + *x = StreamClose{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamClose) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamClose) ProtoMessage() {} + +func (x *StreamClose) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamClose.ProtoReflect.Descriptor instead. +func (*StreamClose) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{11} +} + +func (x *StreamClose) GetStreamId() []byte { + if x != nil { + return x.StreamId + } + return nil +} + +// control message to create a contract +// platform sends a CreateContractResult +type CreateContract struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ulid + DestinationId []byte `protobuf:"bytes,1,opt,name=destination_id,json=destinationId,proto3,oneof" json:"destination_id,omitempty"` + TransferByteCount uint64 `protobuf:"varint,2,opt,name=transfer_byte_count,json=transferByteCount,proto3" json:"transfer_byte_count,omitempty"` + // restrict the contract to a companion of an existing open contract + Companion bool `protobuf:"varint,3,opt,name=companion,proto3" json:"companion,omitempty"` + // ulids + UsedContractIds [][]byte `protobuf:"bytes,4,rep,name=used_contract_ids,json=usedContractIds,proto3" json:"used_contract_ids,omitempty"` + // ulids + IntermediaryIds [][]byte `protobuf:"bytes,5,rep,name=intermediary_ids,json=intermediaryIds,proto3" json:"intermediary_ids,omitempty"` + // ulid + StreamId []byte `protobuf:"bytes,6,opt,name=stream_id,json=streamId,proto3,oneof" json:"stream_id,omitempty"` + // stream will be used when intermediary_ids or stream_id is set + // in the case of wanting to use streams with no intermediaries, set `force_stream=true` + ForceStream *bool `protobuf:"varint,7,opt,name=force_stream,json=forceStream,proto3,oneof" json:"force_stream,omitempty"` +} + +func (x *CreateContract) Reset() { + *x = CreateContract{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CreateContract) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateContract) ProtoMessage() {} + +func (x *CreateContract) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateContract.ProtoReflect.Descriptor instead. +func (*CreateContract) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{12} +} + +func (x *CreateContract) GetDestinationId() []byte { + if x != nil { + return x.DestinationId + } + return nil +} + +func (x *CreateContract) GetTransferByteCount() uint64 { + if x != nil { + return x.TransferByteCount + } + return 0 +} + +func (x *CreateContract) GetCompanion() bool { + if x != nil { + return x.Companion + } + return false +} + +func (x *CreateContract) GetUsedContractIds() [][]byte { + if x != nil { + return x.UsedContractIds + } + return nil +} + +func (x *CreateContract) GetIntermediaryIds() [][]byte { + if x != nil { + return x.IntermediaryIds + } + return nil +} + +func (x *CreateContract) GetStreamId() []byte { + if x != nil { + return x.StreamId + } + return nil +} + +func (x *CreateContract) GetForceStream() bool { + if x != nil && x.ForceStream != nil { + return *x.ForceStream + } + return false +} + +type CreateContractResult struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Error *ContractError `protobuf:"varint,1,opt,name=error,proto3,enum=bringyour.ContractError,oneof" json:"error,omitempty"` + // the public contract to share with the transfer destination + // note this may contain a `stream_id` which directs the client to switch to a stream + Contract *Contract `protobuf:"bytes,2,opt,name=contract,proto3,oneof" json:"contract,omitempty"` + // echo the `CreateContract` to allow associating the result with the request + CreateContract *CreateContract `protobuf:"bytes,3,opt,name=create_contract,json=createContract,proto3,oneof" json:"create_contract,omitempty"` +} + +func (x *CreateContractResult) Reset() { + *x = CreateContractResult{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CreateContractResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateContractResult) ProtoMessage() {} + +func (x *CreateContractResult) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateContractResult.ProtoReflect.Descriptor instead. +func (*CreateContractResult) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{13} +} + +func (x *CreateContractResult) GetError() ContractError { + if x != nil && x.Error != nil { + return *x.Error + } + return ContractError_NoPermission +} + +func (x *CreateContractResult) GetContract() *Contract { + if x != nil { + return x.Contract + } + return nil +} + +func (x *CreateContractResult) GetCreateContract() *CreateContract { + if x != nil { + return x.CreateContract + } + return nil +} + +// append this message inline a pack to enable the connection +type Contract struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ulid + StoredContractBytes []byte `protobuf:"bytes,1,opt,name=stored_contract_bytes,json=storedContractBytes,proto3" json:"stored_contract_bytes,omitempty"` + // `stored_contract_bytes` signed with the `provider_secret_key` + StoredContractHmac []byte `protobuf:"bytes,2,opt,name=stored_contract_hmac,json=storedContractHmac,proto3" json:"stored_contract_hmac,omitempty"` + // the client must always verify the provide mode by matching the `provider_secret_key` for that mode to the hmac + ProvideMode ProvideMode `protobuf:"varint,3,opt,name=provide_mode,json=provideMode,proto3,enum=bringyour.ProvideMode" json:"provide_mode,omitempty"` +} + +func (x *Contract) Reset() { + *x = Contract{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Contract) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Contract) ProtoMessage() {} + +func (x *Contract) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Contract.ProtoReflect.Descriptor instead. +func (*Contract) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{14} +} + +func (x *Contract) GetStoredContractBytes() []byte { + if x != nil { + return x.StoredContractBytes + } + return nil +} + +func (x *Contract) GetStoredContractHmac() []byte { + if x != nil { + return x.StoredContractHmac + } + return nil +} + +func (x *Contract) GetProvideMode() ProvideMode { + if x != nil { + return x.ProvideMode + } + return ProvideMode_None +} + +// stream_id replaces the source_id and destination_id +type StoredContract struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContractId []byte `protobuf:"bytes,1,opt,name=contract_id,json=contractId,proto3" json:"contract_id,omitempty"` + TransferByteCount uint64 `protobuf:"varint,2,opt,name=transfer_byte_count,json=transferByteCount,proto3" json:"transfer_byte_count,omitempty"` + // ulid + SourceId []byte `protobuf:"bytes,3,opt,name=source_id,json=sourceId,proto3,oneof" json:"source_id,omitempty"` + // ulid + DestinationId []byte `protobuf:"bytes,4,opt,name=destination_id,json=destinationId,proto3,oneof" json:"destination_id,omitempty"` + // ulid + StreamId []byte `protobuf:"bytes,5,opt,name=stream_id,json=streamId,proto3,oneof" json:"stream_id,omitempty"` + // relative priority + // 0 (missing) is least priority + // any positive priority value is 100% more than 0 + // generally only positive priority data will matter for the community payouts, + // and priority will scale the payout amount, + // so it is in a provider's interest to prioritize data with positive priority + Priority *uint32 `protobuf:"varint,6,opt,name=priority,proto3,oneof" json:"priority,omitempty"` +} + +func (x *StoredContract) Reset() { + *x = StoredContract{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StoredContract) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StoredContract) ProtoMessage() {} + +func (x *StoredContract) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StoredContract.ProtoReflect.Descriptor instead. +func (*StoredContract) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{15} +} + +func (x *StoredContract) GetContractId() []byte { + if x != nil { + return x.ContractId + } + return nil +} + +func (x *StoredContract) GetTransferByteCount() uint64 { + if x != nil { + return x.TransferByteCount + } + return 0 +} + +func (x *StoredContract) GetSourceId() []byte { + if x != nil { + return x.SourceId + } + return nil +} + +func (x *StoredContract) GetDestinationId() []byte { + if x != nil { + return x.DestinationId + } + return nil +} + +func (x *StoredContract) GetStreamId() []byte { + if x != nil { + return x.StreamId + } + return nil +} + +func (x *StoredContract) GetPriority() uint32 { + if x != nil && x.Priority != nil { + return *x.Priority + } + return 0 +} + +// control message +type CloseContract struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContractId []byte `protobuf:"bytes,1,opt,name=contract_id,json=contractId,proto3" json:"contract_id,omitempty"` + AckedByteCount uint64 `protobuf:"varint,2,opt,name=acked_byte_count,json=ackedByteCount,proto3" json:"acked_byte_count,omitempty"` + UnackedByteCount uint64 `protobuf:"varint,3,opt,name=unacked_byte_count,json=unackedByteCount,proto3" json:"unacked_byte_count,omitempty"` + Checkpoint bool `protobuf:"varint,4,opt,name=checkpoint,proto3" json:"checkpoint,omitempty"` +} + +func (x *CloseContract) Reset() { + *x = CloseContract{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CloseContract) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CloseContract) ProtoMessage() {} + +func (x *CloseContract) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[16] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CloseContract.ProtoReflect.Descriptor instead. +func (*CloseContract) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{16} +} + +func (x *CloseContract) GetContractId() []byte { + if x != nil { + return x.ContractId + } + return nil +} + +func (x *CloseContract) GetAckedByteCount() uint64 { + if x != nil { + return x.AckedByteCount + } + return 0 +} + +func (x *CloseContract) GetUnackedByteCount() uint64 { + if x != nil { + return x.UnackedByteCount + } + return 0 +} + +func (x *CloseContract) GetCheckpoint() bool { + if x != nil { + return x.Checkpoint + } + return false +} + +// peer auditing +// the acl will block contracts between between two parties with bad audits of each other +type PeerAudit struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ulid + PeerId []byte `protobuf:"bytes,1,opt,name=peer_id,json=peerId,proto3" json:"peer_id,omitempty"` + // number of seconds in the audit + Duration uint64 `protobuf:"varint,2,opt,name=duration,proto3" json:"duration,omitempty"` + Abuse bool `protobuf:"varint,3,opt,name=abuse,proto3" json:"abuse,omitempty"` + BadContractCount uint64 `protobuf:"varint,4,opt,name=bad_contract_count,json=badContractCount,proto3" json:"bad_contract_count,omitempty"` + DiscardedByteCount uint64 `protobuf:"varint,5,opt,name=discarded_byte_count,json=discardedByteCount,proto3" json:"discarded_byte_count,omitempty"` + DiscardedCount uint64 `protobuf:"varint,6,opt,name=discarded_count,json=discardedCount,proto3" json:"discarded_count,omitempty"` + BadMessageByteCount uint64 `protobuf:"varint,7,opt,name=bad_message_byte_count,json=badMessageByteCount,proto3" json:"bad_message_byte_count,omitempty"` + BadMessageCount uint64 `protobuf:"varint,8,opt,name=bad_message_count,json=badMessageCount,proto3" json:"bad_message_count,omitempty"` + SendByteCount uint64 `protobuf:"varint,9,opt,name=send_byte_count,json=sendByteCount,proto3" json:"send_byte_count,omitempty"` + SendCount uint64 `protobuf:"varint,10,opt,name=send_count,json=sendCount,proto3" json:"send_count,omitempty"` + ResendByteCount uint64 `protobuf:"varint,11,opt,name=resend_byte_count,json=resendByteCount,proto3" json:"resend_byte_count,omitempty"` + ResendCount uint64 `protobuf:"varint,12,opt,name=resend_count,json=resendCount,proto3" json:"resend_count,omitempty"` + // ulid + StreamId []byte `protobuf:"bytes,13,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` +} + +func (x *PeerAudit) Reset() { + *x = PeerAudit{} + if protoimpl.UnsafeEnabled { + mi := &file_transfer_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PeerAudit) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PeerAudit) ProtoMessage() {} + +func (x *PeerAudit) ProtoReflect() protoreflect.Message { + mi := &file_transfer_proto_msgTypes[17] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PeerAudit.ProtoReflect.Descriptor instead. +func (*PeerAudit) Descriptor() ([]byte, []int) { + return file_transfer_proto_rawDescGZIP(), []int{17} +} + +func (x *PeerAudit) GetPeerId() []byte { + if x != nil { + return x.PeerId + } + return nil +} + +func (x *PeerAudit) GetDuration() uint64 { + if x != nil { + return x.Duration + } + return 0 +} + +func (x *PeerAudit) GetAbuse() bool { + if x != nil { + return x.Abuse + } + return false +} + +func (x *PeerAudit) GetBadContractCount() uint64 { + if x != nil { + return x.BadContractCount + } + return 0 +} + +func (x *PeerAudit) GetDiscardedByteCount() uint64 { + if x != nil { + return x.DiscardedByteCount + } + return 0 +} + +func (x *PeerAudit) GetDiscardedCount() uint64 { + if x != nil { + return x.DiscardedCount + } + return 0 +} + +func (x *PeerAudit) GetBadMessageByteCount() uint64 { + if x != nil { + return x.BadMessageByteCount + } + return 0 +} + +func (x *PeerAudit) GetBadMessageCount() uint64 { + if x != nil { + return x.BadMessageCount + } + return 0 +} + +func (x *PeerAudit) GetSendByteCount() uint64 { + if x != nil { + return x.SendByteCount + } + return 0 +} + +func (x *PeerAudit) GetSendCount() uint64 { + if x != nil { + return x.SendCount + } + return 0 +} + +func (x *PeerAudit) GetResendByteCount() uint64 { + if x != nil { + return x.ResendByteCount + } + return 0 +} + +func (x *PeerAudit) GetResendCount() uint64 { + if x != nil { + return x.ResendCount + } + return 0 +} + +func (x *PeerAudit) GetStreamId() []byte { + if x != nil { + return x.StreamId + } + return nil +} + +var File_transfer_proto protoreflect.FileDescriptor + +var file_transfer_proto_rawDesc = []byte{ + 0x0a, 0x0e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x09, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, 0x72, 0x1a, 0x0b, 0x66, 0x72, 0x61, + 0x6d, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xad, 0x01, 0x0a, 0x0c, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x65, 0x72, 0x50, 0x61, 0x74, 0x68, 0x12, 0x20, 0x0a, 0x09, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x08, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x64, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x48, 0x01, 0x52, 0x0d, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x20, 0x0a, 0x09, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x02, 0x52, 0x08, 0x73, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x49, 0x64, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x64, 0x65, 0x73, 0x74, + 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x73, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x22, 0x75, 0x0a, 0x0d, 0x54, 0x72, 0x61, 0x6e, + 0x73, 0x66, 0x65, 0x72, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x12, 0x3c, 0x0a, 0x0d, 0x74, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x65, 0x72, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x17, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, 0x72, 0x2e, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x65, 0x72, 0x50, 0x61, 0x74, 0x68, 0x52, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x65, 0x72, 0x50, 0x61, 0x74, 0x68, 0x12, 0x26, 0x0a, 0x05, 0x66, 0x72, 0x61, 0x6d, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, + 0x75, 0x72, 0x2e, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x52, 0x05, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x22, + 0x55, 0x0a, 0x15, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x65, 0x72, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x12, 0x3c, 0x0a, 0x0d, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x66, 0x65, 0x72, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x17, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, 0x72, 0x2e, 0x54, 0x72, 0x61, 0x6e, + 0x73, 0x66, 0x65, 0x72, 0x50, 0x61, 0x74, 0x68, 0x52, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, + 0x65, 0x72, 0x50, 0x61, 0x74, 0x68, 0x22, 0x8e, 0x01, 0x0a, 0x1e, 0x46, 0x69, 0x6c, 0x74, 0x65, + 0x72, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x46, 0x72, 0x61, 0x6d, 0x65, + 0x57, 0x69, 0x74, 0x68, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x12, 0x3c, 0x0a, 0x0d, 0x74, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x65, 0x72, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x17, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, 0x72, 0x2e, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x65, 0x72, 0x50, 0x61, 0x74, 0x68, 0x52, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x65, 0x72, 0x50, 0x61, 0x74, 0x68, 0x12, 0x2e, 0x0a, 0x05, 0x66, 0x72, 0x61, 0x6d, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, + 0x75, 0x72, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x65, 0x64, 0x46, 0x72, 0x61, 0x6d, 0x65, + 0x52, 0x05, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x22, 0x92, 0x02, 0x0a, 0x04, 0x50, 0x61, 0x63, 0x6b, + 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x49, 0x64, 0x12, + 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x64, + 0x12, 0x27, 0x0a, 0x0f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x75, 0x6d, + 0x62, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x73, 0x65, 0x71, 0x75, 0x65, + 0x6e, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x61, + 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x68, 0x65, 0x61, 0x64, 0x12, 0x28, 0x0a, + 0x06, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, + 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, 0x72, 0x2e, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x52, + 0x06, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x63, 0x6b, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x6e, 0x61, 0x63, 0x6b, 0x12, 0x3c, 0x0a, 0x0e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, 0x72, 0x2e, + 0x46, 0x72, 0x61, 0x6d, 0x65, 0x48, 0x00, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, + 0x74, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x22, 0x5f, 0x0a, 0x0c, + 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x12, 0x3c, 0x0a, 0x0e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, 0x72, + 0x2e, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x48, 0x00, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, + 0x63, 0x74, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x22, 0x63, 0x0a, + 0x03, 0x41, 0x63, 0x6b, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, + 0x63, 0x65, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x76, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x76, 0x65, 0x22, 0x5f, 0x0a, 0x04, 0x41, 0x75, 0x74, 0x68, 0x12, 0x15, 0x0a, 0x06, 0x62, 0x79, + 0x5f, 0x6a, 0x77, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x62, 0x79, 0x4a, 0x77, + 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x70, 0x70, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x70, 0x70, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x69, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, + 0x65, 0x49, 0x64, 0x22, 0x34, 0x0a, 0x07, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x12, 0x29, + 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x62, + 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, 0x72, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, + 0x4b, 0x65, 0x79, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0x66, 0x0a, 0x0a, 0x50, 0x72, 0x6f, + 0x76, 0x69, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, + 0x72, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6d, + 0x6f, 0x64, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x5f, 0x73, + 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x10, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, + 0x79, 0x22, 0x98, 0x01, 0x0a, 0x0a, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4f, 0x70, 0x65, 0x6e, + 0x12, 0x20, 0x0a, 0x09, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x08, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x88, + 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x01, 0x52, 0x0d, 0x64, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x1b, + 0x0a, 0x09, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x08, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x49, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x64, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x22, 0x2a, 0x0a, 0x0b, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x73, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, + 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x49, 0x64, 0x22, 0xdd, 0x02, 0x0a, 0x0e, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x12, 0x2a, 0x0a, 0x0e, 0x64, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0d, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x2e, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x65, 0x72, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x11, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x42, 0x79, + 0x74, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x61, + 0x6e, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, + 0x61, 0x6e, 0x69, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, + 0x52, 0x0f, 0x75, 0x73, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x49, 0x64, + 0x73, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x72, + 0x79, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0f, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x72, 0x79, 0x49, 0x64, 0x73, 0x12, 0x20, 0x0a, 0x09, + 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x48, + 0x01, 0x52, 0x08, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x26, + 0x0a, 0x0c, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x08, 0x48, 0x02, 0x52, 0x0b, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x88, 0x01, 0x01, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x64, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x73, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x66, 0x6f, 0x72, 0x63, + 0x65, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x22, 0xf5, 0x01, 0x0a, 0x14, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x12, 0x33, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x18, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, 0x72, 0x2e, 0x43, 0x6f, 0x6e, + 0x74, 0x72, 0x61, 0x63, 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x88, 0x01, 0x01, 0x12, 0x34, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, + 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, + 0x79, 0x6f, 0x75, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x48, 0x01, 0x52, + 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x88, 0x01, 0x01, 0x12, 0x47, 0x0a, 0x0f, + 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, 0x6f, 0x75, + 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, + 0x48, 0x02, 0x52, 0x0e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, + 0x63, 0x74, 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, + 0x0b, 0x0a, 0x09, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x42, 0x12, 0x0a, 0x10, + 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, + 0x22, 0xab, 0x01, 0x0a, 0x08, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x12, 0x32, 0x0a, + 0x15, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, + 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x13, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x42, 0x79, 0x74, 0x65, + 0x73, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x61, 0x63, 0x74, 0x5f, 0x68, 0x6d, 0x61, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x12, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x48, + 0x6d, 0x61, 0x63, 0x12, 0x39, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x5f, 0x6d, + 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x62, 0x72, 0x69, 0x6e, + 0x67, 0x79, 0x6f, 0x75, 0x72, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x4d, 0x6f, 0x64, + 0x65, 0x52, 0x0b, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x22, 0xae, + 0x02, 0x0a, 0x0e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, + 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, + 0x49, 0x64, 0x12, 0x2e, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x5f, 0x62, + 0x79, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x11, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x42, 0x79, 0x74, 0x65, 0x43, 0x6f, 0x75, + 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x09, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x08, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, + 0x64, 0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x01, 0x52, 0x0d, + 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x88, 0x01, 0x01, + 0x12, 0x20, 0x0a, 0x09, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0c, 0x48, 0x02, 0x52, 0x08, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x49, 0x64, 0x88, + 0x01, 0x01, 0x12, 0x1f, 0x0a, 0x08, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0d, 0x48, 0x03, 0x52, 0x08, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, + 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, + 0x64, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x69, 0x64, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x22, + 0xa8, 0x01, 0x0a, 0x0d, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, + 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, + 0x49, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, + 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x61, 0x63, + 0x6b, 0x65, 0x64, 0x42, 0x79, 0x74, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2c, 0x0a, 0x12, + 0x75, 0x6e, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x75, + 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x75, 0x6e, 0x61, 0x63, 0x6b, 0x65, + 0x64, 0x42, 0x79, 0x74, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x68, + 0x65, 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, + 0x63, 0x68, 0x65, 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x22, 0xf3, 0x03, 0x0a, 0x09, 0x50, + 0x65, 0x65, 0x72, 0x41, 0x75, 0x64, 0x69, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x70, 0x65, 0x65, 0x72, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x70, 0x65, 0x65, 0x72, 0x49, + 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, + 0x05, 0x61, 0x62, 0x75, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x61, 0x62, + 0x75, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x62, 0x61, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x61, 0x63, 0x74, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x10, 0x62, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x43, 0x6f, 0x75, 0x6e, + 0x74, 0x12, 0x30, 0x0a, 0x14, 0x64, 0x69, 0x73, 0x63, 0x61, 0x72, 0x64, 0x65, 0x64, 0x5f, 0x62, + 0x79, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x12, 0x64, 0x69, 0x73, 0x63, 0x61, 0x72, 0x64, 0x65, 0x64, 0x42, 0x79, 0x74, 0x65, 0x43, 0x6f, + 0x75, 0x6e, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x64, 0x69, 0x73, 0x63, 0x61, 0x72, 0x64, 0x65, 0x64, + 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x64, 0x69, + 0x73, 0x63, 0x61, 0x72, 0x64, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x33, 0x0a, 0x16, + 0x62, 0x61, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, + 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x13, 0x62, 0x61, + 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x79, 0x74, 0x65, 0x43, 0x6f, 0x75, 0x6e, + 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x62, 0x61, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x62, 0x61, + 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x26, 0x0a, + 0x0f, 0x73, 0x65, 0x6e, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x73, 0x65, 0x6e, 0x64, 0x42, 0x79, 0x74, 0x65, + 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x6e, 0x64, 0x5f, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x73, 0x65, 0x6e, 0x64, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x64, 0x5f, 0x62, + 0x79, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x0f, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x64, 0x42, 0x79, 0x74, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, + 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x64, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x18, 0x0c, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x64, 0x43, 0x6f, + 0x75, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, + 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x49, 0x64, + 0x2a, 0x52, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, + 0x08, 0x0a, 0x04, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x4e, 0x65, 0x74, + 0x77, 0x6f, 0x72, 0x6b, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x46, 0x72, 0x69, 0x65, 0x6e, 0x64, + 0x73, 0x41, 0x6e, 0x64, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, + 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x10, 0x04, 0x2a, 0x45, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x10, 0x0a, 0x0c, 0x4e, 0x6f, 0x50, 0x65, 0x72, 0x6d, 0x69, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x49, 0x6e, 0x73, 0x75, 0x66, + 0x66, 0x69, 0x63, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x10, 0x01, + 0x12, 0x09, 0x0a, 0x05, 0x53, 0x65, 0x74, 0x75, 0x70, 0x10, 0x02, 0x42, 0x27, 0x5a, 0x25, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x62, 0x72, 0x69, 0x6e, 0x67, 0x79, + 0x6f, 0x75, 0x72, 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_transfer_proto_rawDescOnce sync.Once + file_transfer_proto_rawDescData = file_transfer_proto_rawDesc +) + +func file_transfer_proto_rawDescGZIP() []byte { + file_transfer_proto_rawDescOnce.Do(func() { + file_transfer_proto_rawDescData = protoimpl.X.CompressGZIP(file_transfer_proto_rawDescData) + }) + return file_transfer_proto_rawDescData +} + +var file_transfer_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_transfer_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_transfer_proto_goTypes = []interface{}{ + (ProvideMode)(0), // 0: bringyour.ProvideMode + (ContractError)(0), // 1: bringyour.ContractError + (*TransferPath)(nil), // 2: bringyour.TransferPath + (*TransferFrame)(nil), // 3: bringyour.TransferFrame + (*FilteredTransferFrame)(nil), // 4: bringyour.FilteredTransferFrame + (*FilteredTransferFrameWithFrame)(nil), // 5: bringyour.FilteredTransferFrameWithFrame + (*Pack)(nil), // 6: bringyour.Pack + (*FilteredPack)(nil), // 7: bringyour.FilteredPack + (*Ack)(nil), // 8: bringyour.Ack + (*Auth)(nil), // 9: bringyour.Auth + (*Provide)(nil), // 10: bringyour.Provide + (*ProvideKey)(nil), // 11: bringyour.ProvideKey + (*StreamOpen)(nil), // 12: bringyour.StreamOpen + (*StreamClose)(nil), // 13: bringyour.StreamClose + (*CreateContract)(nil), // 14: bringyour.CreateContract + (*CreateContractResult)(nil), // 15: bringyour.CreateContractResult + (*Contract)(nil), // 16: bringyour.Contract + (*StoredContract)(nil), // 17: bringyour.StoredContract + (*CloseContract)(nil), // 18: bringyour.CloseContract + (*PeerAudit)(nil), // 19: bringyour.PeerAudit + (*Frame)(nil), // 20: bringyour.Frame + (*FilteredFrame)(nil), // 21: bringyour.FilteredFrame +} +var file_transfer_proto_depIdxs = []int32{ + 2, // 0: bringyour.TransferFrame.transfer_path:type_name -> bringyour.TransferPath + 20, // 1: bringyour.TransferFrame.frame:type_name -> bringyour.Frame + 2, // 2: bringyour.FilteredTransferFrame.transfer_path:type_name -> bringyour.TransferPath + 2, // 3: bringyour.FilteredTransferFrameWithFrame.transfer_path:type_name -> bringyour.TransferPath + 21, // 4: bringyour.FilteredTransferFrameWithFrame.frame:type_name -> bringyour.FilteredFrame + 20, // 5: bringyour.Pack.frames:type_name -> bringyour.Frame + 20, // 6: bringyour.Pack.contract_frame:type_name -> bringyour.Frame + 20, // 7: bringyour.FilteredPack.contract_frame:type_name -> bringyour.Frame + 11, // 8: bringyour.Provide.keys:type_name -> bringyour.ProvideKey + 0, // 9: bringyour.ProvideKey.mode:type_name -> bringyour.ProvideMode + 1, // 10: bringyour.CreateContractResult.error:type_name -> bringyour.ContractError + 16, // 11: bringyour.CreateContractResult.contract:type_name -> bringyour.Contract + 14, // 12: bringyour.CreateContractResult.create_contract:type_name -> bringyour.CreateContract + 0, // 13: bringyour.Contract.provide_mode:type_name -> bringyour.ProvideMode + 14, // [14:14] is the sub-list for method output_type + 14, // [14:14] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name +} + +func init() { file_transfer_proto_init() } +func file_transfer_proto_init() { + if File_transfer_proto != nil { + return + } + file_frame_proto_init() + if !protoimpl.UnsafeEnabled { + file_transfer_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TransferPath); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TransferFrame); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FilteredTransferFrame); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FilteredTransferFrameWithFrame); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Pack); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FilteredPack); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Ack); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Auth); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Provide); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProvideKey); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamOpen); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamClose); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CreateContract); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CreateContractResult); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Contract); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StoredContract); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CloseContract); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_transfer_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PeerAudit); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_transfer_proto_msgTypes[0].OneofWrappers = []interface{}{} + file_transfer_proto_msgTypes[4].OneofWrappers = []interface{}{} + file_transfer_proto_msgTypes[5].OneofWrappers = []interface{}{} + file_transfer_proto_msgTypes[10].OneofWrappers = []interface{}{} + file_transfer_proto_msgTypes[12].OneofWrappers = []interface{}{} + file_transfer_proto_msgTypes[13].OneofWrappers = []interface{}{} + file_transfer_proto_msgTypes[15].OneofWrappers = []interface{}{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_transfer_proto_rawDesc, + NumEnums: 2, + NumMessages: 18, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_transfer_proto_goTypes, + DependencyIndexes: file_transfer_proto_depIdxs, + EnumInfos: file_transfer_proto_enumTypes, + MessageInfos: file_transfer_proto_msgTypes, + }.Build() + File_transfer_proto = out.File + file_transfer_proto_rawDesc = nil + file_transfer_proto_goTypes = nil + file_transfer_proto_depIdxs = nil +} diff --git a/protocol/transfer.proto b/protocol/transfer.proto index 23d4f1e..30f534a 100644 --- a/protocol/transfer.proto +++ b/protocol/transfer.proto @@ -3,7 +3,7 @@ package bringyour; import "frame.proto"; -option go_package = "bringyour.com/protocol"; +option go_package = "github.com/bringyour/connect/protocol"; // changes 2024-08: diff --git a/provider/go.mod b/provider/go.mod deleted file mode 100644 index a39ee16..0000000 --- a/provider/go.mod +++ /dev/null @@ -1,25 +0,0 @@ -module bringyour.com/connect/provider - -go 1.22.0 - -replace bringyour.com/connect v0.0.0 => ../connect - -replace bringyour.com/protocol v0.0.0 => ../protocol/build/bringyour.com/protocol - -require ( - bringyour.com/connect v0.0.0 - bringyour.com/protocol v0.0.0 - github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 - github.com/golang-jwt/jwt/v5 v5.2.0 - golang.org/x/term v0.15.0 -) - -require ( - github.com/golang/glog v1.2.1 // indirect - github.com/google/gopacket v1.1.19 // indirect - github.com/gorilla/websocket v1.5.0 // indirect - github.com/oklog/ulid/v2 v2.1.0 // indirect - golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect - golang.org/x/sys v0.15.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect -) diff --git a/provider/main.go b/provider/main.go index 77170e3..43f2b2d 100644 --- a/provider/main.go +++ b/provider/main.go @@ -1,34 +1,32 @@ package main import ( - "context" - "fmt" - "os" - "syscall" - "net/http" - "encoding/json" - "errors" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "syscall" - "golang.org/x/term" + "golang.org/x/term" - "github.com/docopt/docopt-go" + "github.com/docopt/docopt-go" - gojwt "github.com/golang-jwt/jwt/v5" + gojwt "github.com/golang-jwt/jwt/v5" - "bringyour.com/connect" - "bringyour.com/protocol" + "github.com/bringyour/connect/connect" + "github.com/bringyour/connect/protocol" ) const DefaultApiUrl = "https://api.bringyour.com" const DefaultConnectUrl = "wss://connect.bringyour.com" - const LocalVersion = "0.0.0-local" - func main() { - usage := fmt.Sprintf( - `Connect provider. + usage := fmt.Sprintf( + `Connect provider. The default urls are: api_url: %s @@ -47,274 +45,259 @@ Options: --user_auth= --password= -p --port= Listen port [default: 80].`, - DefaultApiUrl, - DefaultConnectUrl, - ) - - opts, err := docopt.ParseArgs(usage, os.Args[1:], RequireVersion()) - if err != nil { - panic(err) - } - - if provide_, _ := opts.Bool("provide"); provide_ { - provide(opts) - } + DefaultApiUrl, + DefaultConnectUrl, + ) + + opts, err := docopt.ParseArgs(usage, os.Args[1:], RequireVersion()) + if err != nil { + panic(err) + } + + if provide_, _ := opts.Bool("provide"); provide_ { + provide(opts) + } } - func provide(opts docopt.Opts) { - port, _ := opts.Int("--port") - - var apiUrl string - if apiUrlAny := opts["--api_url"]; apiUrlAny != nil { - apiUrl = apiUrlAny.(string) - } else { - apiUrl = DefaultApiUrl - } - - var connectUrl string - if connectUrlAny := opts["--connect_url"]; connectUrlAny != nil { - connectUrl = connectUrlAny.(string) - } else { - connectUrl = DefaultConnectUrl - } - - - cancelCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - - event := connect.NewEventWithContext(cancelCtx) - event.SetOnSignals(syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM) - - ctx := event.Ctx() - - - byClientJwt, clientId := provideAuth(ctx, apiUrl, opts) - - instanceId := connect.NewId() - - clientOob := connect.NewApiOutOfBandControl(ctx, byClientJwt, apiUrl) - connectClient := connect.NewClientWithDefaults(ctx, clientId, clientOob) - - // routeManager := connect.NewRouteManager(connectClient) - // contractManager := connect.NewContractManagerWithDefaults(connectClient) - // connectClient.Setup(routeManager, contractManager) - // go connectClient.Run() - - fmt.Printf("client_id: %s\n", clientId) - fmt.Printf("instance_id: %s\n", instanceId) - - auth := &connect.ClientAuth{ - ByJwt: byClientJwt, - // ClientId: clientId, - InstanceId: instanceId, - AppVersion: RequireVersion(), - } - connect.NewPlatformTransportWithDefaults(ctx, connectUrl, auth, connectClient.RouteManager()) - // go platformTransport.Run(connectClient.RouteManager()) - - localUserNat := connect.NewLocalUserNatWithDefaults(ctx, clientId.String()) - remoteUserNatProvider := connect.NewRemoteUserNatProviderWithDefaults(connectClient, localUserNat) - - provideModes := map[protocol.ProvideMode]bool{ - protocol.ProvideMode_Public: true, - protocol.ProvideMode_Network: true, - } - connectClient.ContractManager().SetProvideModes(provideModes) - - - fmt.Printf( - "Status %s on *:%d\n", - RequireVersion(), - port, - ) - - - statusServer := &http.Server{ - Addr: fmt.Sprintf(":%d", port), - Handler: &Status{}, - } - - go func() { - defer cancel() - err := statusServer.ListenAndServe() - if err != nil { - fmt.Printf("status error: %s\n", err) - } - }() - - select { - case <- ctx.Done(): - } - - statusServer.Shutdown(ctx) - - remoteUserNatProvider.Close() - localUserNat.Close() - connectClient.Cancel() - - // exit - os.Exit(0) + port, _ := opts.Int("--port") + + var apiUrl string + if apiUrlAny := opts["--api_url"]; apiUrlAny != nil { + apiUrl = apiUrlAny.(string) + } else { + apiUrl = DefaultApiUrl + } + + var connectUrl string + if connectUrlAny := opts["--connect_url"]; connectUrlAny != nil { + connectUrl = connectUrlAny.(string) + } else { + connectUrl = DefaultConnectUrl + } + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + event := connect.NewEventWithContext(cancelCtx) + event.SetOnSignals(syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM) + + ctx := event.Ctx() + + byClientJwt, clientId := provideAuth(ctx, apiUrl, opts) + + instanceId := connect.NewId() + + clientOob := connect.NewApiOutOfBandControl(ctx, byClientJwt, apiUrl) + connectClient := connect.NewClientWithDefaults(ctx, clientId, clientOob) + + // routeManager := connect.NewRouteManager(connectClient) + // contractManager := connect.NewContractManagerWithDefaults(connectClient) + // connectClient.Setup(routeManager, contractManager) + // go connectClient.Run() + + fmt.Printf("client_id: %s\n", clientId) + fmt.Printf("instance_id: %s\n", instanceId) + + auth := &connect.ClientAuth{ + ByJwt: byClientJwt, + // ClientId: clientId, + InstanceId: instanceId, + AppVersion: RequireVersion(), + } + connect.NewPlatformTransportWithDefaults(ctx, connectUrl, auth, connectClient.RouteManager()) + // go platformTransport.Run(connectClient.RouteManager()) + + localUserNat := connect.NewLocalUserNatWithDefaults(ctx, clientId.String()) + remoteUserNatProvider := connect.NewRemoteUserNatProviderWithDefaults(connectClient, localUserNat) + + provideModes := map[protocol.ProvideMode]bool{ + protocol.ProvideMode_Public: true, + protocol.ProvideMode_Network: true, + } + connectClient.ContractManager().SetProvideModes(provideModes) + + fmt.Printf( + "Status %s on *:%d\n", + RequireVersion(), + port, + ) + + statusServer := &http.Server{ + Addr: fmt.Sprintf(":%d", port), + Handler: &Status{}, + } + + go func() { + defer cancel() + err := statusServer.ListenAndServe() + if err != nil { + fmt.Printf("status error: %s\n", err) + } + }() + + select { + case <-ctx.Done(): + } + + statusServer.Shutdown(ctx) + + remoteUserNatProvider.Close() + localUserNat.Close() + connectClient.Cancel() + + // exit + os.Exit(0) } - func provideAuth(ctx context.Context, apiUrl string, opts docopt.Opts) (byClientJwt string, clientId connect.Id) { - userAuth := opts["--user_auth"].(string) - - var password string - if passwordAny := opts["--password"]; passwordAny != nil { - password = passwordAny.(string) - } else { - fmt.Print("Enter password: ") - passwordBytes, err := term.ReadPassword(int(syscall.Stdin)) - if err != nil { - panic(err) - } - password = string(passwordBytes) - fmt.Printf("\n") - } - - // fmt.Printf("userAuth='%s'; password='%s'\n", userAuth, password) - - api := connect.NewBringYourApiWithContext(ctx, apiUrl) - - loginCallback, loginChannel := connect.NewBlockingApiCallback[*connect.AuthLoginWithPasswordResult]() - - loginArgs := &connect.AuthLoginWithPasswordArgs{ - UserAuth: userAuth, - Password: password, - } - - api.AuthLoginWithPassword(loginArgs, loginCallback) - - var loginResult connect.ApiCallbackResult[*connect.AuthLoginWithPasswordResult] - select { - case <- ctx.Done(): - os.Exit(0) - case loginResult = <- loginChannel: - } - - if loginResult.Error != nil { - panic(loginResult.Error) - } - if loginResult.Result.Error != nil { - panic(fmt.Errorf("%s", loginResult.Result.Error.Message)) - } - if loginResult.Result.VerificationRequired != nil { - panic(fmt.Errorf("Verification required for %s. Use the app or web to complete account setup.", loginResult.Result.VerificationRequired.UserAuth)) - } - - api.SetByJwt(loginResult.Result.Network.ByJwt) - - - authClientCallback, authClientChannel := connect.NewBlockingApiCallback[*connect.AuthNetworkClientResult]() - - authClientArgs := &connect.AuthNetworkClientArgs{ - Description: fmt.Sprintf("provider %s", RequireVersion()), - DeviceSpec: "", - } - - api.AuthNetworkClient(authClientArgs, authClientCallback) - - var authClientResult connect.ApiCallbackResult[*connect.AuthNetworkClientResult] - select { - case <- ctx.Done(): - os.Exit(0) - case authClientResult = <- authClientChannel: - } - - if authClientResult.Error != nil { - panic(authClientResult.Error) - } - if authClientResult.Result.Error != nil { - panic(fmt.Errorf("%s", authClientResult.Result.Error.Message)) - } - - byClientJwt = authClientResult.Result.ByClientJwt - - // parse the clientId - parser := gojwt.NewParser() - token, _, err := parser.ParseUnverified(byClientJwt, gojwt.MapClaims{}) - if err != nil { - panic(err) - } - - claims := token.Claims.(gojwt.MapClaims) - - clientId, err = connect.ParseId(claims["client_id"].(string)) - if err != nil { - panic(err) - } - - return + userAuth := opts["--user_auth"].(string) + + var password string + if passwordAny := opts["--password"]; passwordAny != nil { + password = passwordAny.(string) + } else { + fmt.Print("Enter password: ") + passwordBytes, err := term.ReadPassword(int(syscall.Stdin)) + if err != nil { + panic(err) + } + password = string(passwordBytes) + fmt.Printf("\n") + } + + // fmt.Printf("userAuth='%s'; password='%s'\n", userAuth, password) + + api := connect.NewBringYourApiWithContext(ctx, apiUrl) + + loginCallback, loginChannel := connect.NewBlockingApiCallback[*connect.AuthLoginWithPasswordResult]() + + loginArgs := &connect.AuthLoginWithPasswordArgs{ + UserAuth: userAuth, + Password: password, + } + + api.AuthLoginWithPassword(loginArgs, loginCallback) + + var loginResult connect.ApiCallbackResult[*connect.AuthLoginWithPasswordResult] + select { + case <-ctx.Done(): + os.Exit(0) + case loginResult = <-loginChannel: + } + + if loginResult.Error != nil { + panic(loginResult.Error) + } + if loginResult.Result.Error != nil { + panic(fmt.Errorf("%s", loginResult.Result.Error.Message)) + } + if loginResult.Result.VerificationRequired != nil { + panic(fmt.Errorf("Verification required for %s. Use the app or web to complete account setup.", loginResult.Result.VerificationRequired.UserAuth)) + } + + api.SetByJwt(loginResult.Result.Network.ByJwt) + + authClientCallback, authClientChannel := connect.NewBlockingApiCallback[*connect.AuthNetworkClientResult]() + + authClientArgs := &connect.AuthNetworkClientArgs{ + Description: fmt.Sprintf("provider %s", RequireVersion()), + DeviceSpec: "", + } + + api.AuthNetworkClient(authClientArgs, authClientCallback) + + var authClientResult connect.ApiCallbackResult[*connect.AuthNetworkClientResult] + select { + case <-ctx.Done(): + os.Exit(0) + case authClientResult = <-authClientChannel: + } + + if authClientResult.Error != nil { + panic(authClientResult.Error) + } + if authClientResult.Result.Error != nil { + panic(fmt.Errorf("%s", authClientResult.Result.Error.Message)) + } + + byClientJwt = authClientResult.Result.ByClientJwt + + // parse the clientId + parser := gojwt.NewParser() + token, _, err := parser.ParseUnverified(byClientJwt, gojwt.MapClaims{}) + if err != nil { + panic(err) + } + + claims := token.Claims.(gojwt.MapClaims) + + clientId, err = connect.ParseId(claims["client_id"].(string)) + if err != nil { + panic(err) + } + + return } - - - type Status struct { } func (self *Status) ServeHTTP(w http.ResponseWriter, r *http.Request) { - type WarpStatusResult struct { - Version string `json:"version,omitempty"` - ConfigVersion string `json:"config_version,omitempty"` - Status string `json:"status"` - ClientAddress string `json:"client_address,omitempty"` - Host string `json:"host"` - } - - result := &WarpStatusResult{ - Version: RequireVersion(), - ConfigVersion: RequireConfigVersion(), - Status: "ok", - Host: RequireHost(), - } - - responseJson, err := json.Marshal(result) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/json") - w.Write(responseJson) + type WarpStatusResult struct { + Version string `json:"version,omitempty"` + ConfigVersion string `json:"config_version,omitempty"` + Status string `json:"status"` + ClientAddress string `json:"client_address,omitempty"` + Host string `json:"host"` + } + + result := &WarpStatusResult{ + Version: RequireVersion(), + ConfigVersion: RequireConfigVersion(), + Status: "ok", + Host: RequireHost(), + } + + responseJson, err := json.Marshal(result) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(responseJson) } - func Host() (string, error) { - host := os.Getenv("WARP_HOST") - if host != "" { - return host, nil - } - host, err := os.Hostname() - if err == nil { - return host, nil - } - return "", errors.New("WARP_HOST not set") + host := os.Getenv("WARP_HOST") + if host != "" { + return host, nil + } + host, err := os.Hostname() + if err == nil { + return host, nil + } + return "", errors.New("WARP_HOST not set") } - func RequireHost() string { - host, err := Host() - if err != nil { - panic(err) - } - return host + host, err := Host() + if err != nil { + panic(err) + } + return host } - func RequireVersion() string { - if version := os.Getenv("WARP_VERSION"); version != "" { - return version - } - return LocalVersion + if version := os.Getenv("WARP_VERSION"); version != "" { + return version + } + return LocalVersion } - func RequireConfigVersion() string { - if version := os.Getenv("WARP_CONFIG_VERSION"); version != "" { - return version - } - return LocalVersion + if version := os.Getenv("WARP_CONFIG_VERSION"); version != "" { + return version + } + return LocalVersion } - diff --git a/test-ip-local/go.mod b/test-ip-local/go.mod index 9c4ad6e..1215c68 100644 --- a/test-ip-local/go.mod +++ b/test-ip-local/go.mod @@ -4,11 +4,11 @@ go 1.22.0 replace bringyour.com/connect v0.0.0 => ../connect -replace bringyour.com/protocol v0.0.0 => ../protocol/build/bringyour.com/protocol +replace github.com/bringyour/connect/protocol v0.0.0 => ../protocol/build/github.com/bringyour/connect/protocol require ( bringyour.com/connect v0.0.0 // indirect - bringyour.com/protocol v0.0.0 // indirect + github.com/bringyour/connect/protocol v0.0.0 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/oklog/ulid/v2 v2.1.0 // indirect golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect diff --git a/test-ip-local/main.go b/test-ip-local/main.go index f3b3c5e..cbd3c75 100644 --- a/test-ip-local/main.go +++ b/test-ip-local/main.go @@ -1,125 +1,114 @@ package main // ip-local -// reads raw packets from the interface, writes +// reads raw packets from the interface, writes import ( - "context" - "os" - "os/signal" - "syscall" + "context" + "os" + "os/signal" + "syscall" - "golang.org/x/sys/unix" + "golang.org/x/sys/unix" - "bringyour.com/connect" - "bringyour.com/protocol" + "bringyour.com/connect" + "github.com/bringyour/connect/protocol" ) - // see https://android.googlesource.com/platform/development/+/master/samples/ToyVpn/server/linux/ToyVpnServer.cpp - - - - func main() { - log := connect.LogFn(connect.LogLevelInfo, "test-ip-local") - connect.GlobalLogLevel = connect.LogLevelDebug - - interfaceName := "tun0" - clientPath := connect.Path{ - ClientId: connect.Id{}, - StreamId: connect.Id{}, - } - - ctx, cancel := context.WithCancel(context.Background()) - - - // unix.O_NONBLOCK - fd, err := unix.Open("/dev/net/tun", unix.O_RDWR, 0) - if err != nil { - panic(err) - } - defer unix.Close(fd) - - ifreq, err := unix.NewIfreq(interfaceName) - if err != nil { - panic(err) - } - // flags - // IFF_TUN taptun interface - // IFF_NO_PI no protocol info header on each read packet - ifreq.SetUint16(unix.IFF_TUN | unix.IFF_NO_PI) - - if err := unix.IoctlIfreq(fd, unix.TUNSETIFF, ifreq); err != nil { - panic(err) - } - - signalNotify := make(chan os.Signal) - signal.Notify(signalNotify, syscall.SIGTERM, syscall.SIGQUIT) - go func() { - <- signalNotify - cancel() - }() - - // FIXME listen for quit signal - - - // FIXME create remote nat - - remoteUserNat := connect.NewRemoteUserNatWithDefaults(ctx) - go remoteUserNat.Run() - - remoteUserNat.AddSendPacketCallback(func(destination connect.Path, packet []byte) { - // FIXME log the packet - if destination == clientPath { - //log("write %x", packet) - _, err := unix.Write(fd, packet) - if err != nil { - panic(err) - } - } else { - log("discard %x", packet) - } - }) - - - // FIXME start read loop (log packets) - // FIXME start write loop (log packets) - - buffer := make([]byte, 2048) - - for { - select { - case <- ctx.Done(): - return - default: - } - n, err := unix.Read(fd, buffer) - if err != nil { - panic(err) - } - if 0 < n { - packet := make([]byte, n) - copy(packet, buffer[0:n]) - // FIXME log - remoteUserNat.Receive(clientPath, protocol.ProvideMode_PUBLIC, packet) - } - } - - - // var ifr struct { - // name [16]byte - // flags uint16 - // _ [22]byte - // } - // copy(ifr.name[:], "tun0") - // ifr.flags = unix.IFF_TUN | unix.IFF_NO_PI - // _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), unix.TUNSETIFF, uintptr(unsafe.Pointer(&ifr))) - // if errno != 0 { - // log.Println("err to syscall:", errno) - // return - // } + log := connect.LogFn(connect.LogLevelInfo, "test-ip-local") + connect.GlobalLogLevel = connect.LogLevelDebug + + interfaceName := "tun0" + clientPath := connect.Path{ + ClientId: connect.Id{}, + StreamId: connect.Id{}, + } + + ctx, cancel := context.WithCancel(context.Background()) + + // unix.O_NONBLOCK + fd, err := unix.Open("/dev/net/tun", unix.O_RDWR, 0) + if err != nil { + panic(err) + } + defer unix.Close(fd) + + ifreq, err := unix.NewIfreq(interfaceName) + if err != nil { + panic(err) + } + // flags + // IFF_TUN taptun interface + // IFF_NO_PI no protocol info header on each read packet + ifreq.SetUint16(unix.IFF_TUN | unix.IFF_NO_PI) + + if err := unix.IoctlIfreq(fd, unix.TUNSETIFF, ifreq); err != nil { + panic(err) + } + + signalNotify := make(chan os.Signal) + signal.Notify(signalNotify, syscall.SIGTERM, syscall.SIGQUIT) + go func() { + <-signalNotify + cancel() + }() + + // FIXME listen for quit signal + + // FIXME create remote nat + + remoteUserNat := connect.NewRemoteUserNatWithDefaults(ctx) + go remoteUserNat.Run() + + remoteUserNat.AddSendPacketCallback(func(destination connect.Path, packet []byte) { + // FIXME log the packet + if destination == clientPath { + //log("write %x", packet) + _, err := unix.Write(fd, packet) + if err != nil { + panic(err) + } + } else { + log("discard %x", packet) + } + }) + + // FIXME start read loop (log packets) + // FIXME start write loop (log packets) + + buffer := make([]byte, 2048) + + for { + select { + case <-ctx.Done(): + return + default: + } + n, err := unix.Read(fd, buffer) + if err != nil { + panic(err) + } + if 0 < n { + packet := make([]byte, n) + copy(packet, buffer[0:n]) + // FIXME log + remoteUserNat.Receive(clientPath, protocol.ProvideMode_PUBLIC, packet) + } + } + + // var ifr struct { + // name [16]byte + // flags uint16 + // _ [22]byte + // } + // copy(ifr.name[:], "tun0") + // ifr.flags = unix.IFF_TUN | unix.IFF_NO_PI + // _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), unix.TUNSETIFF, uintptr(unsafe.Pointer(&ifr))) + // if errno != 0 { + // log.Println("err to syscall:", errno) + // return + // } } - -