Socket buffer controls to solve bufferbloat.

README.md

@@ -118,6 +118,13 @@ This field also has no effect if `AdminUID` isn't a valid UID or is empty.
 `KeepAlive` is the number of seconds to tell the OS to wait after no activity before sending TCP KeepAlive probes to the
 upstream proxy server. Zero or negative value disables it. Default is 0 (disabled).
 
+`LoopbackTcpSendBuffer` is the number of bytes to use for the tcp loopback send buffer. Use a low value like 4096 for a server-to-server bridge.
+
+`LoopbackTcpReceiveBuffer` is the number of bytes to use for the tcp loopback receive buffer. Use a low value like 4096 for a server-to-server bridge.
+
+These 2 options are not normally needed except when setting up a tcp server-to-server bridge using a shadowsocks or similar tcp server in the `ProxyBook` to reduce tcp performance degradation due to bufferbloat across the bridge.
+
+
 ### Client
 
 `UID` is your UID in base64.
@@ -179,6 +186,14 @@ more detectable as a proxy, but it will make the Cloak client detect internet in
 data, after which the connection will be closed by Cloak. Cloak will not enforce any timeout on TCP connections after it
 is established.
 
+`LoopbackTcpSendBuffer` is the number of bytes to use for the tcp loopback send buffer. Use a low value like 4096 to reduce upload bufferbloat on client.
+
+`LoopbackTcpReceiveBuffer` is the number of bytes to use for the tcp loopback receive buffer. Use a low value like 4096 to reduce download bufferbloat on client if the server is very close (low ping).
+
+`RemoteTcpSendBuffer` is the number of bytes to use for the tcp remote send buffer. Use a low value like 4096 for a server-to-server bridge.
+
+`RemoteTcpReceiveBuffer` is the number of bytes to use for the tcp remote receive buffer. Use a low value like 4096 for a server-to-server bridge.
+
 ## Setup
 
 ### Server

cmd/ck-client/ck-client.go

@@ -8,10 +8,10 @@ import (
 	"encoding/binary"
 	"flag"
 	"fmt"
+	"github.com/cbeuw/Cloak/internal/common"
 	"net"
 	"os"
-
-	"github.com/cbeuw/Cloak/internal/common"
+	"syscall"
 
 	"github.com/cbeuw/Cloak/internal/client"
 	mux "github.com/cbeuw/Cloak/internal/multiplex"
@@ -154,7 +154,35 @@ func main() {
 
 	var seshMaker func() *mux.Session
 
-	d := &net.Dialer{Control: protector, KeepAlive: remoteConfig.KeepAlive}
+	control := func(network string, address string, rawConn syscall.RawConn) error {
+		sendBufferSize := remoteConfig.TcpSendBuffer
+		receiveBufferSize := remoteConfig.TcpReceiveBuffer
+
+		err := rawConn.Control(func(fd uintptr) {
+			if sendBufferSize > 0 {
+				log.Debugf("Setting remote connection tcp send buffer: %d", sendBufferSize)
+				err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, sendBufferSize)
+				if err != nil {
+					log.Errorf("setsocketopt SO_SNDBUF: %s\n", err)
+				}
+			}
+
+			if receiveBufferSize > 0 {
+				log.Debugf("Setting remote connection tcp receive buffer: %d", receiveBufferSize)
+				err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, receiveBufferSize)
+				if err != nil {
+					log.Errorf("setsocketopt SO_RCVBUF: %s\n", err)
+				}
+			}
+		})
+		if err != nil {
+			panic(err)
+		}
+
+		return protector(network, address, rawConn)
+	}
+
+	d := &net.Dialer{Control: control, KeepAlive: remoteConfig.KeepAlive}
 
 	if adminUID != nil {
 		log.Infof("API base is %v", localConfig.LocalAddr)
@@ -199,8 +227,43 @@ func main() {
 	} else {
 		listener, err := net.Listen("tcp", localConfig.LocalAddr)
 		if err != nil {
-			log.Fatal(err)
+			panic(err)
 		}
+
+		tcpListener, ok := listener.(*net.TCPListener)
+		if !ok {
+			panic("Unknown listener type")
+		}
+
+		syscallConn, err := tcpListener.SyscallConn()
+		if err != nil {
+			panic(err)
+		}
+
+		sendBufferSize := localConfig.TcpSendBuffer
+		receiveBufferSize := localConfig.TcpReceiveBuffer
+
+		err = syscallConn.Control(func(fd uintptr) {
+			if sendBufferSize > 0 {
+				log.Debugf("Setting remote connection tcp send buffer: %d", sendBufferSize)
+				err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, sendBufferSize)
+				if err != nil {
+					log.Errorf("setsocketopt SO_SNDBUF: %s\n", err)
+				}
+			}
+
+			if receiveBufferSize > 0 {
+				log.Debugf("Setting remote connection tcp receive buffer: %d", receiveBufferSize)
+				err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, receiveBufferSize)
+				if err != nil {
+					log.Errorf("setsocketopt SO_RCVBUF: %s\n", err)
+				}
+			}
+		})
+		if err != nil {
+			panic(err)
+		}
+
 		client.RouteTCP(listener, localConfig.Timeout, remoteConfig.Singleplex, seshMaker)
 	}
 }

internal/client/state.go

@@ -33,27 +33,35 @@ type RawConfig struct {
 	RemotePort       string   // jsonOptional
 	AlternativeNames []string // jsonOptional
 	// defaults set in ProcessRawConfig
-	UDP           bool   // nullable
-	BrowserSig    string // nullable
-	Transport     string // nullable
-	CDNOriginHost string // nullable
-	CDNWsUrlPath  string // nullable
-	StreamTimeout int    // nullable
-	KeepAlive     int    // nullable
+	UDP                      bool   // nullable
+	BrowserSig               string // nullable
+	Transport                string // nullable
+	CDNOriginHost            string // nullable
+	CDNWsUrlPath             string // nullable
+	StreamTimeout            int    // nullable
+	KeepAlive                int    // nullable
+	LoopbackTcpSendBuffer    int    // nullable
+	LoopbackTcpReceiveBuffer int    // nullable
+	RemoteTcpSendBuffer      int    // nullable
+	RemoteTcpReceiveBuffer   int    // nullable
 }
 
 type RemoteConnConfig struct {
-	Singleplex     bool
-	NumConn        int
-	KeepAlive      time.Duration
-	RemoteAddr     string
-	TransportMaker func() Transport
+	Singleplex       bool
+	NumConn          int
+	KeepAlive        time.Duration
+	RemoteAddr       string
+	TransportMaker   func() Transport
+	TcpSendBuffer    int
+	TcpReceiveBuffer int
 }
 
 type LocalConnConfig struct {
-	LocalAddr      string
-	Timeout        time.Duration
-	MockDomainList []string
+	LocalAddr        string
+	Timeout          time.Duration
+	MockDomainList   []string
+	TcpSendBuffer    int
+	TcpReceiveBuffer int
 }
 
 type AuthInfo struct {
@@ -83,7 +91,16 @@ func ssvToJson(ssv string) (ret []byte) {
 		r = strings.Replace(r, `\;`, `;`, -1)
 		return r
 	}
-	unquoted := []string{"NumConn", "StreamTimeout", "KeepAlive", "UDP"}
+	unquoted := []string{
+		"NumConn",
+		"StreamTimeout",
+		"KeepAlive",
+		"UDP",
+		"LoopbackTcpSendBuffer",
+		"LoopbackTcpReceiveBuffer",
+		"RemoteTcpSendBuffer",
+		"RemoteTcpReceiveBuffer",
+	}
 	lines := strings.Split(unescape(ssv), ";")
 	ret = []byte("{")
 	for _, ln := range lines {
@@ -277,5 +294,21 @@ func (raw *RawConfig) ProcessRawConfig(worldState common.WorldState) (local Loca
 		local.Timeout = time.Duration(raw.StreamTimeout) * time.Second
 	}
 
+	if raw.LoopbackTcpSendBuffer > 0 {
+		local.TcpSendBuffer = raw.LoopbackTcpSendBuffer
+	}
+
+	if raw.LoopbackTcpReceiveBuffer > 0 {
+		local.TcpReceiveBuffer = raw.LoopbackTcpReceiveBuffer
+	}
+
+	if raw.RemoteTcpSendBuffer > 0 {
+		remote.TcpSendBuffer = raw.RemoteTcpSendBuffer
+	}
+
+	if raw.RemoteTcpReceiveBuffer > 0 {
+		remote.TcpReceiveBuffer = raw.RemoteTcpReceiveBuffer
+	}
+
 	return
 }

internal/common/platformfd_darwin.go

@@ -0,0 +1,8 @@
+//go:build darwin
+// +build darwin
+
+package common
+
+func Platformfd(fd uintptr) int {
+	return int(fd)
+}

internal/common/platformfd_linux.go

@@ -0,0 +1,8 @@
+//go:build linux
+// +build linux
+
+package common
+
+func Platformfd(fd uintptr) int {
+	return int(fd)
+}

internal/common/platformfd_windows.go

@@ -0,0 +1,10 @@
+//go:build windows
+// +build windows
+
+package common
+
+import "syscall"
+
+func Platformfd(fd uintptr) syscall.Handle {
+	return syscall.Handle(fd)
+}

internal/server/state.go

@@ -9,22 +9,26 @@ import (
 	"net"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/cbeuw/Cloak/internal/common"
 	"github.com/cbeuw/Cloak/internal/server/usermanager"
+	log "github.com/sirupsen/logrus"
 )
 
 type RawConfig struct {
-	ProxyBook    map[string][]string
-	BindAddr     []string
-	BypassUID    [][]byte
-	RedirAddr    string
-	PrivateKey   []byte
-	AdminUID     []byte
-	DatabasePath string
-	KeepAlive    int
-	CncMode      bool
+	ProxyBook                map[string][]string
+	BindAddr                 []string
+	BypassUID                [][]byte
+	RedirAddr                string
+	PrivateKey               []byte
+	AdminUID                 []byte
+	DatabasePath             string
+	KeepAlive                int
+	CncMode                  bool
+	LoopbackTcpSendBuffer    int
+	LoopbackTcpReceiveBuffer int
 }
 
 // State type stores the global state of the program
@@ -156,10 +160,46 @@ func InitState(preParse RawConfig, worldState common.WorldState) (sta *State, er
 		sta.Panel = MakeUserPanel(manager)
 	}
 
+	dialerControl := func(network, address string, c syscall.RawConn) error {
+		if !strings.HasPrefix(network, "tcp") {
+			return nil
+		}
+
+		ips, err := net.LookupHost(strings.Split(address, ":")[0])
+		if err != nil {
+			return err
+		}
+
+		for _, ipString := range ips {
+			ip := net.ParseIP(ipString)
+			if !ip.IsLoopback() {
+				return nil
+			}
+		}
+
+		return c.Control(func(fd uintptr) {
+			if preParse.LoopbackTcpSendBuffer > 0 {
+				log.Debugf("Setting loopback connection tcp send buffer: %d", preParse.LoopbackTcpSendBuffer)
+				err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, preParse.LoopbackTcpSendBuffer)
+				if err != nil {
+					log.Errorf("setsocketopt SO_SNDBUF: %s\n", err)
+				}
+			}
+
+			if preParse.LoopbackTcpReceiveBuffer > 0 {
+				log.Debugf("Setting loopback connection tcp receive buffer: %d", preParse.LoopbackTcpReceiveBuffer)
+				err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, preParse.LoopbackTcpReceiveBuffer)
+				if err != nil {
+					log.Errorf("setsocketopt SO_RCVBUF: %s\n", err)
+				}
+			}
+		})
+	}
+
 	if preParse.KeepAlive <= 0 {
-		sta.ProxyDialer = &net.Dialer{KeepAlive: -1}
+		sta.ProxyDialer = &net.Dialer{KeepAlive: -1, Control: dialerControl}
 	} else {
-		sta.ProxyDialer = &net.Dialer{KeepAlive: time.Duration(preParse.KeepAlive) * time.Second}
+		sta.ProxyDialer = &net.Dialer{KeepAlive: time.Duration(preParse.KeepAlive) * time.Second, Control: dialerControl}
 	}
 
 	sta.RedirHost, sta.RedirPort, err = parseRedirAddr(preParse.RedirAddr)