Skip to content

Commit

Permalink
conf: use server-http-tls to specify http security | tidb-test=pr/2248 (
Browse files Browse the repository at this point in the history 
#403)

Signed-off-by: xhe <[email protected]>
  • Loading branch information
@xhebox
xhebox authored Nov 18, 2023
1 parent d148588 commit 8712ca0
Show file tree
Hide file tree
Showing 8 changed files with 61 additions and 52 deletions.
12 changes: 7 additions & 5 deletions conf/proxy.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,6 @@ graceful-close-conn-timeout = 15
# server object:
# 1. requires: cert/key or auto-certs(generate a temporary cert, mostly for testing)
# 2. optionally: ca will enable server-side client verification. If skip-ca is true with non-empty ca, server will only verify clients if it can provide any cert. Otherwise, clients must provide a cert.
# peer object:
# 1. requires: cert/key/ca or auto-certs
# 2. useless/forbid: skip-ca

# client object
[security.cluster-tls]
Expand All @@ -98,12 +95,17 @@ graceful-close-conn-timeout = 15

# client object
[security.sql-tls]
# access to other components like TiDB or PD, will use this
# access to TiDB SQL(4000) port will use this
skip-ca = true

# server object
[security.server-tls]
# proxy SQL or HTTP port will use this
# proxy SQL port will use this
# auto-certs = true

# server object
[security.server-http-tls]
# proxy HTTP port will use this
# auto-certs = true

[metrics]
Expand Down
13 changes: 7 additions & 6 deletions lib/config/proxy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,10 +113,10 @@ func (c TLSConfig) HasCA() bool {
}

type Security struct {
ServerTLS TLSConfig `yaml:"server-tls,omitempty" toml:"server-tls,omitempty" json:"server-tls,omitempty"`
PeerTLS TLSConfig `yaml:"peer-tls,omitempty" toml:"peer-tls,omitempty" json:"peer-tls,omitempty"`
ClusterTLS TLSConfig `yaml:"cluster-tls,omitempty" toml:"cluster-tls,omitempty" json:"cluster-tls,omitempty"`
SQLTLS TLSConfig `yaml:"sql-tls,omitempty" toml:"sql-tls,omitempty" json:"sql-tls,omitempty"`
ServerSQLTLS TLSConfig `yaml:"server-tls,omitempty" toml:"server-tls,omitempty" json:"server-tls,omitempty"`
ServerHTTPTLS TLSConfig `yaml:"server-http-tls,omitempty" toml:"server-http-tls,omitempty" json:"server-http-tls,omitempty"`
ClusterTLS TLSConfig `yaml:"cluster-tls,omitempty" toml:"cluster-tls,omitempty" json:"cluster-tls,omitempty"`
SQLTLS TLSConfig `yaml:"sql-tls,omitempty" toml:"sql-tls,omitempty" json:"sql-tls,omitempty"`
}

func DefaultKeepAlive() (frontend, backendHealthy, backendUnhealthy KeepAlive) {
Expand Down Expand Up @@ -153,8 +153,8 @@ func NewConfig() *Config {

cfg.Advance.IgnoreWrongNamespace = true
cfg.Security.SQLTLS.MinTLSVersion = "1.1"
cfg.Security.PeerTLS.MinTLSVersion = "1.1"
cfg.Security.ServerTLS.MinTLSVersion = "1.1"
cfg.Security.ServerSQLTLS.MinTLSVersion = "1.1"
cfg.Security.ServerHTTPTLS.MinTLSVersion = "1.1"
cfg.Security.ClusterTLS.MinTLSVersion = "1.1"

return &cfg
Expand Down Expand Up @@ -184,6 +184,7 @@ func (cfg *Config) Check() error {
if cfg.Proxy.ConnBufferSize > 0 && (cfg.Proxy.ConnBufferSize > 16*1024*1024 || cfg.Proxy.ConnBufferSize < 1024) {
return errors.Wrapf(ErrInvalidConfigValue, "conn-buffer-size must be between 1K and 16M")
}

return nil
}

Expand Down
4 changes: 2 additions & 2 deletions lib/config/proxy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,13 @@ var testProxyConfig = Config{
},
},
Security: Security{
ServerTLS: TLSConfig{
ServerSQLTLS: TLSConfig{
CA: "a",
Cert: "b",
Key: "c",
AutoCerts: true,
},
PeerTLS: TLSConfig{
ServerHTTPTLS: TLSConfig{
CA: "a",
Cert: "b",
Key: "c",
Expand Down
44 changes: 22 additions & 22 deletions pkg/manager/cert/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,14 @@ const (
// Currently, all the namespaces share the same certs but there might be per-namespace
// certs in the future.
type CertManager struct {
serverTLS *security.CertInfo // client / proxyctl -> proxy
serverTLSConfig atomic.Pointer[tls.Config]
peerTLS *security.CertInfo // proxy -> proxy
peerTLSConfig atomic.Pointer[tls.Config]
clusterTLS *security.CertInfo // proxy -> pd / tidb status port
clusterTLSConfig atomic.Pointer[tls.Config]
sqlTLS *security.CertInfo // proxy -> tidb sql port
sqlTLSConfig atomic.Pointer[tls.Config]
serverSQLTLS *security.CertInfo // client -> proxy
serverSQLTLSConfig atomic.Pointer[tls.Config]
serverHTTPTLS *security.CertInfo // proxyctl -> proxy
serverHTTPTLSConfig atomic.Pointer[tls.Config]
clusterTLS *security.CertInfo // proxy -> pd / tidb status port
clusterTLSConfig atomic.Pointer[tls.Config]
sqlTLS *security.CertInfo // proxy -> tidb sql port
sqlTLSConfig atomic.Pointer[tls.Config]

cancel context.CancelFunc
wg waitgroup.WaitGroup
Expand All @@ -51,8 +51,8 @@ func NewCertManager() *CertManager {
// cfgch can be set to nil for the serverless tier because it has no config manager.
func (cm *CertManager) Init(cfg *config.Config, logger *zap.Logger, cfgch <-chan *config.Config) error {
cm.logger = logger
cm.serverTLS = security.NewCert(true)
cm.peerTLS = security.NewCert(false)
cm.serverSQLTLS = security.NewCert(true)
cm.serverHTTPTLS = security.NewCert(true)
cm.clusterTLS = security.NewCert(false)
cm.sqlTLS = security.NewCert(false)
cm.setConfig(cfg)
Expand All @@ -67,8 +67,8 @@ func (cm *CertManager) Init(cfg *config.Config, logger *zap.Logger, cfgch <-chan
}

func (cm *CertManager) setConfig(cfg *config.Config) {
cm.serverTLS.SetConfig(cfg.Security.ServerTLS)
cm.peerTLS.SetConfig(cfg.Security.PeerTLS)
cm.serverSQLTLS.SetConfig(cfg.Security.ServerSQLTLS)
cm.serverHTTPTLS.SetConfig(cfg.Security.ServerHTTPTLS)
cm.clusterTLS.SetConfig(cfg.Security.ClusterTLS)
cm.sqlTLS.SetConfig(cfg.Security.SQLTLS)
}
Expand All @@ -77,16 +77,16 @@ func (cm *CertManager) SetRetryInterval(interval time.Duration) {
cm.retryInterval.Store(int64(interval))
}

func (cm *CertManager) ServerTLS() *tls.Config {
return cm.serverTLSConfig.Load()
func (cm *CertManager) ServerSQLTLS() *tls.Config {
return cm.serverSQLTLSConfig.Load()
}

func (cm *CertManager) ClusterTLS() *tls.Config {
return cm.clusterTLSConfig.Load()
func (cm *CertManager) ServerHTTPTLS() *tls.Config {
return cm.serverHTTPTLSConfig.Load()
}

func (cm *CertManager) PeerTLS() *tls.Config {
return cm.peerTLSConfig.Load()
func (cm *CertManager) ClusterTLS() *tls.Config {
return cm.clusterTLSConfig.Load()
}

func (cm *CertManager) SQLTLS() *tls.Config {
Expand Down Expand Up @@ -122,15 +122,15 @@ func (cm *CertManager) reloadLoop(ctx context.Context, cfgch <-chan *config.Conf
// If any error happens, we still continue and use the old cert.
func (cm *CertManager) reload() error {
errs := make([]error, 0, 4)
if tlsConfig, err := cm.serverTLS.Reload(cm.logger); err != nil {
if tlsConfig, err := cm.serverSQLTLS.Reload(cm.logger); err != nil {
errs = append(errs, err)
} else {
cm.serverTLSConfig.Store(tlsConfig)
cm.serverSQLTLSConfig.Store(tlsConfig)
}
if tlsConfig, err := cm.peerTLS.Reload(cm.logger); err != nil {
if tlsConfig, err := cm.serverHTTPTLS.Reload(cm.logger); err != nil {
errs = append(errs, err)
} else {
cm.peerTLSConfig.Store(tlsConfig)
cm.serverHTTPTLSConfig.Store(tlsConfig)
}
if tlsConfig, err := cm.clusterTLS.Reload(cm.logger); err != nil {
errs = append(errs, err)
Expand Down
32 changes: 19 additions & 13 deletions pkg/manager/cert/manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,38 +73,44 @@ func TestInit(t *testing.T) {
{
name: "empty",
check: func(t *testing.T, cm *CertManager) {
require.Nil(t, cm.ServerTLS())
require.Nil(t, cm.ServerSQLTLS())
require.Nil(t, cm.ClusterTLS())
require.Nil(t, cm.PeerTLS())
require.Nil(t, cm.ServerHTTPTLS())
require.Nil(t, cm.SQLTLS())
},
},
{
name: "server config",
cfg: config.Config{
Security: config.Security{
ServerTLS: config.TLSConfig{AutoCerts: true},
ServerSQLTLS: config.TLSConfig{AutoCerts: true},
ServerHTTPTLS: config.TLSConfig{AutoCerts: true},
ClusterTLS: config.TLSConfig{AutoCerts: true},
SQLTLS: config.TLSConfig{AutoCerts: true},
},
},
check: func(t *testing.T, cm *CertManager) {
require.Nil(t, cm.ClusterTLS())
require.Nil(t, cm.PeerTLS())
require.Nil(t, cm.SQLTLS())
require.NotNil(t, cm.ServerTLS())
require.NotNil(t, cm.ServerHTTPTLS())
require.NotNil(t, cm.ServerSQLTLS())
},
},
{
name: "client config",
cfg: config.Config{
Security: config.Security{
SQLTLS: config.TLSConfig{SkipCA: true},
ServerSQLTLS: config.TLSConfig{SkipCA: true},
ServerHTTPTLS: config.TLSConfig{SkipCA: true},
ClusterTLS: config.TLSConfig{SkipCA: true},
SQLTLS: config.TLSConfig{SkipCA: true},
},
},
check: func(t *testing.T, cm *CertManager) {
require.Nil(t, cm.ClusterTLS())
require.Nil(t, cm.PeerTLS())
require.Nil(t, cm.ServerTLS())
require.NotNil(t, cm.ClusterTLS())
require.NotNil(t, cm.SQLTLS())
require.Nil(t, cm.ServerHTTPTLS())
require.Nil(t, cm.ServerSQLTLS())
},
},
{
Expand Down Expand Up @@ -159,7 +165,7 @@ func TestRotate(t *testing.T) {
cfg := &config.Config{
Workdir: tmpdir,
Security: config.Security{
ServerTLS: config.TLSConfig{
ServerSQLTLS: config.TLSConfig{
Cert: certPath,
Key: keyPath,
},
Expand Down Expand Up @@ -270,7 +276,7 @@ func TestRotate(t *testing.T) {
}
require.NoError(t, certMgr.Init(cfg, lg, nil))

stls := certMgr.ServerTLS()
stls := certMgr.ServerSQLTLS()
ctls := certMgr.SQLTLS()

// pre reloading test
Expand Down Expand Up @@ -335,7 +341,7 @@ func TestBidirectional(t *testing.T) {
cfg := &config.Config{
Workdir: tmpdir,
Security: config.Security{
ServerTLS: config.TLSConfig{
ServerSQLTLS: config.TLSConfig{
Cert: certPath1,
Key: keyPath1,
CA: caPath2,
Expand All @@ -350,7 +356,7 @@ func TestBidirectional(t *testing.T) {

certMgr := NewCertManager()
require.NoError(t, certMgr.Init(cfg, lg, nil))
stls := certMgr.ServerTLS()
stls := certMgr.ServerSQLTLS()
ctls := certMgr.SQLTLS()
clientErr, serverErr := connectWithTLS(ctls, stls)
require.NoError(t, clientErr)
Expand Down
2 changes: 1 addition & 1 deletion pkg/proxy/proxy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ func (s *SQLServer) onConn(ctx context.Context, conn net.Conn, addr string) {
s.mu.connID++
logger := s.logger.With(zap.Uint64("connID", connID), zap.String("client_addr", conn.RemoteAddr().String()),
zap.String("addr", addr))
clientConn := client.NewClientConnection(logger.Named("conn"), conn, s.certMgr.ServerTLS(), s.certMgr.SQLTLS(),
clientConn := client.NewClientConnection(logger.Named("conn"), conn, s.certMgr.ServerSQLTLS(), s.certMgr.SQLTLS(),
s.hsHandler, connID, addr, &backend.BCConfig{
ProxyProtocol: s.mu.proxyProtocol,
RequireBackendTLS: s.mu.requireBackendTLS,
Expand Down
4 changes: 2 additions & 2 deletions pkg/server/api/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ ignore-wrong-namespace = true
[security.server-tls]
min-tls-version = '1.1'
[security.peer-tls]
[security.server-http-tls]
min-tls-version = '1.1'
[security.cluster-tls]
Expand All @@ -76,7 +76,7 @@ max-backups = 3
doHTTP(t, http.MethodGet, "/api/admin/config?format=json", nil, func(t *testing.T, r *http.Response) {
all, err := io.ReadAll(r.Body)
require.NoError(t, err)
require.Equal(t, `{"proxy":{"addr":"0.0.0.0:6000","pd-addrs":"127.0.0.1:2379","require-backend-tls":true,"frontend-keepalive":{"enabled":true},"backend-healthy-keepalive":{"enabled":true,"idle":60000000000,"cnt":5,"intvl":3000000000,"timeout":15000000000},"backend-unhealthy-keepalive":{"enabled":true,"idle":10000000000,"cnt":5,"intvl":1000000000,"timeout":5000000000},"graceful-close-conn-timeout":15},"api":{"addr":"0.0.0.0:3080"},"advance":{"ignore-wrong-namespace":true},"security":{"server-tls":{"min-tls-version":"1.1"},"peer-tls":{"min-tls-version":"1.1"},"cluster-tls":{"min-tls-version":"1.1"},"sql-tls":{"min-tls-version":"1.1"}},"metrics":{"metrics-addr":"","metrics-interval":0},"log":{"encoder":"tidb","level":"info","log-file":{"max-size":300,"max-days":3,"max-backups":3}}}`,
require.Equal(t, `{"proxy":{"addr":"0.0.0.0:6000","pd-addrs":"127.0.0.1:2379","require-backend-tls":true,"frontend-keepalive":{"enabled":true},"backend-healthy-keepalive":{"enabled":true,"idle":60000000000,"cnt":5,"intvl":3000000000,"timeout":15000000000},"backend-unhealthy-keepalive":{"enabled":true,"idle":10000000000,"cnt":5,"intvl":1000000000,"timeout":5000000000},"graceful-close-conn-timeout":15},"api":{"addr":"0.0.0.0:3080"},"advance":{"ignore-wrong-namespace":true},"security":{"server-tls":{"min-tls-version":"1.1"},"server-http-tls":{"min-tls-version":"1.1"},"cluster-tls":{"min-tls-version":"1.1"},"sql-tls":{"min-tls-version":"1.1"}},"metrics":{"metrics-addr":"","metrics-interval":0},"log":{"encoder":"tidb","level":"info","log-file":{"max-size":300,"max-days":3,"max-backups":3}}}`,
string(regexp.MustCompile(`"workdir":"[^"]+",`).ReplaceAll(all, nil)))
require.Equal(t, http.StatusOK, r.StatusCode)
})
Expand Down
2 changes: 1 addition & 1 deletion pkg/server/api/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ func NewServer(cfg config.API, lg *zap.Logger,
}
}

if tlscfg := crtmgr.ServerTLS(); tlscfg != nil {
if tlscfg := crtmgr.ServerHTTPTLS(); tlscfg != nil {
h.listener = tls.NewListener(h.listener, tlscfg)
}

Expand Down

0 comments on commit 8712ca0

Please sign in to comment.