Add support for TLS certificate hot-reload (oliver006#526)

* Hot-reload of TLS server certificate

* Hot-reload of TLS client certificate

* Improve TLS testing with generated certificates

Author: bjosv

.gitignore

@@ -9,3 +9,8 @@ src/
 .idea
 .vscode/
 *.rdb
+contrib/tls/ca.crt
+contrib/tls/ca.key
+contrib/tls/ca.txt
+contrib/tls/redis.crt
+contrib/tls/redis.key

Makefile

@@ -21,7 +21,8 @@ docker-test:
 
 
 .PHONY: test
-test: 
+test:
+	contrib/tls/gen-test-certs.sh
 	TEST_REDIS_URI="redis://redis6:6379" \
 	TEST_REDIS5_URI="redis://redis5:6383" \
 	TEST_REDIS6_URI="redis://redis6:6379" \

contrib/tls/gen-test-certs.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Generate test certificates:
+#
+#   ca.{crt,key}          Self signed CA certificate.
+#   redis.{crt,key}       A certificate with no key usage/policy restrictions.
+
+dir=`dirname $0`
+
+# Generate CA
+openssl genrsa -out ${dir}/ca.key 4096
+openssl req \
+    -x509 -new -nodes -sha256 \
+    -key ${dir}/ca.key \
+    -days 3650 \
+    -subj '/O=redis_exporter/CN=Certificate Authority' \
+    -out ${dir}/ca.crt
+
+# Generate cert
+openssl genrsa -out ${dir}/redis.key 2048
+openssl req \
+    -new -sha256 \
+    -subj "/O=redis_exporter/CN=localhost" \
+    -key ${dir}/redis.key | \
+    openssl x509 \
+        -req -sha256 \
+        -CA ${dir}/ca.crt \
+        -CAkey ${dir}/ca.key \
+        -CAserial ${dir}/ca.txt \
+        -CAcreateserial \
+        -days 3650 \
+        -out ${dir}/redis.crt

exporter/exporter.go

@@ -1,8 +1,6 @@
 package exporter
 
 import (
-	"crypto/tls"
-	"crypto/x509"
 	"fmt"
 	"net/http"
 	"runtime"
@@ -61,8 +59,9 @@ type Options struct {
 	MaxDistinctKeyGroups  int64
 	CountKeys             string
 	LuaScript             []byte
-	ClientCertificates    []tls.Certificate
-	CaCertificates        *x509.CertPool
+	ClientCertFile        string
+	ClientKeyFile         string
+	CaCertFile            string
 	InclSystemMetrics     bool
 	SkipTLSVerification   bool
 	SetClientName         bool

exporter/redis.go

@@ -1,24 +1,23 @@
 package exporter
 
 import (
-	"crypto/tls"
 	"strings"
 
 	"github.com/gomodule/redigo/redis"
 	log "github.com/sirupsen/logrus"
 )
 
 func (e *Exporter) connectToRedis() (redis.Conn, error) {
+	tlsConfig, err := e.CreateClientTLSConfig()
+	if err != nil {
+		return nil, err
+	}
+
 	options := []redis.DialOption{
 		redis.DialConnectTimeout(e.options.ConnectionTimeouts),
 		redis.DialReadTimeout(e.options.ConnectionTimeouts),
 		redis.DialWriteTimeout(e.options.ConnectionTimeouts),
-
-		redis.DialTLSConfig(&tls.Config{
-			InsecureSkipVerify: e.options.SkipTLSVerification,
-			Certificates:       e.options.ClientCertificates,
-			RootCAs:            e.options.CaCertificates,
-		}),
+		redis.DialTLSConfig(tlsConfig),
 	}
 
 	if e.options.User != "" {

exporter/tls.go

@@ -0,0 +1,55 @@
+package exporter
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"io/ioutil"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// CreateClientTLSConfig verifies configured files and return a prepared tls.Config
+func (e *Exporter) CreateClientTLSConfig() (*tls.Config, error) {
+	tlsConfig := tls.Config{
+		InsecureSkipVerify: e.options.SkipTLSVerification,
+	}
+
+	if e.options.ClientCertFile != "" && e.options.ClientKeyFile != "" {
+		cert, err := LoadKeyPair(e.options.ClientCertFile, e.options.ClientKeyFile)
+		if err != nil {
+			return nil, err
+		}
+		tlsConfig.Certificates = []tls.Certificate{*cert}
+	}
+
+	if e.options.CaCertFile != "" {
+		log.Debugf("Load CA cert: %s", e.options.CaCertFile)
+		caCert, err := ioutil.ReadFile(e.options.CaCertFile)
+		if err != nil {
+			return nil, err
+		}
+		certificates := x509.NewCertPool()
+		certificates.AppendCertsFromPEM(caCert)
+		tlsConfig.RootCAs = certificates
+	}
+
+	return &tlsConfig, nil
+}
+
+// GetServerCertificateFunc returns a function for tls.Config.GetCertificate
+func GetServerCertificateFunc(certFile, keyFile string) func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+	return func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+		return LoadKeyPair(certFile, keyFile)
+	}
+}
+
+// LoadKeyPair reads and parses a public/private key pair from a pair of files.
+// The files must contain PEM encoded data.
+func LoadKeyPair(certFile, keyFile string) (*tls.Certificate, error) {
+	log.Debugf("Load key pair: %s %s", certFile, keyFile)
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return nil, err
+	}
+	return &cert, nil
+}

exporter/tls_test.go

@@ -0,0 +1,54 @@
+package exporter
+
+import (
+	"testing"
+)
+
+func TestCreateClientTLSConfig(t *testing.T) {
+	for _, test := range []struct {
+		name          string
+		options       Options
+		expectSuccess bool
+	}{
+		// positive tests
+		{"no_options", Options{}, true},
+		{"skip_verificaton", Options{
+			SkipTLSVerification: true}, true},
+		{"load_client_keypair", Options{
+			ClientCertFile: "../contrib/tls/redis.crt",
+			ClientKeyFile:  "../contrib/tls/redis.key"}, true},
+		{"load_ca_cert", Options{
+			CaCertFile: "../contrib/tls/ca.crt"}, true},
+
+		// negative tests
+		{"nonexisting_client_files", Options{
+			ClientCertFile: "/nonexisting/file",
+			ClientKeyFile:  "/nonexisting/file"}, false},
+		{"nonexisting_ca_file", Options{
+			CaCertFile: "/nonexisting/file"}, false},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			e := getTestExporterWithOptions(test.options)
+
+			_, err := e.CreateClientTLSConfig()
+			if test.expectSuccess && err != nil {
+				t.Errorf("Expected success for test: %s, got err: %s", test.name, err)
+				return
+			}
+		})
+	}
+}
+
+func TestGetServerCertificateFunc(t *testing.T) {
+	// positive test
+	_, err := GetServerCertificateFunc("../contrib/tls/ca.crt", "../contrib/tls/ca.key")(nil)
+	if err != nil {
+		t.Errorf("GetServerCertificateFunc() err: %s", err)
+	}
+
+	// negative test
+	_, err = GetServerCertificateFunc("/nonexisting/file", "/nonexisting/file")(nil)
+	if err == nil {
+		t.Errorf("Expected GetServerCertificateFunc() to fail")
+	}
+}

main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"crypto/tls"
-	"crypto/x509"
 	"flag"
 	"io/ioutil"
 	"net/http"
@@ -120,28 +119,6 @@ func main() {
 		log.Fatalf("Couldn't parse connection timeout duration, err: %s", err)
 	}
 
-	var tlsClientCertificates []tls.Certificate
-	if (*tlsClientKeyFile != "") != (*tlsClientCertFile != "") {
-		log.Fatal("TLS client key file and cert file should both be present")
-	}
-	if *tlsClientKeyFile != "" && *tlsClientCertFile != "" {
-		cert, err := tls.LoadX509KeyPair(*tlsClientCertFile, *tlsClientKeyFile)
-		if err != nil {
-			log.Fatalf("Couldn't load TLS client key pair, err: %s", err)
-		}
-		tlsClientCertificates = append(tlsClientCertificates, cert)
-	}
-
-	var tlsCaCertificates *x509.CertPool
-	if *tlsCaCertFile != "" {
-		caCert, err := ioutil.ReadFile(*tlsCaCertFile)
-		if err != nil {
-			log.Fatalf("Couldn't load TLS Ca certificate, err: %s", err)
-		}
-		tlsCaCertificates = x509.NewCertPool()
-		tlsCaCertificates.AppendCertsFromPEM(caCert)
-	}
-
 	passwordMap := make(map[string]string)
 	if *redisPwd == "" && *redisPwdFile != "" {
 		passwordMap, err = exporter.LoadPwdFile(*redisPwdFile)
@@ -185,8 +162,9 @@ func main() {
 			ExportClientList:      *exportClientList,
 			ExportClientsInclPort: *exportClientPort,
 			SkipTLSVerification:   *skipTLSVerification,
-			ClientCertificates:    tlsClientCertificates,
-			CaCertificates:        tlsCaCertificates,
+			ClientCertFile:        *tlsClientCertFile,
+			ClientKeyFile:         *tlsClientKeyFile,
+			CaCertFile:            *tlsCaCertFile,
 			ConnectionTimeouts:    to,
 			MetricsPath:           *metricPath,
 			RedisMetricsOnly:      *redisMetricsOnly,
@@ -203,11 +181,30 @@ func main() {
 		log.Fatal(err)
 	}
 
+	// Verify that initial client keypair and CA are accepted
+	if (*tlsClientCertFile != "") != (*tlsClientKeyFile != "") {
+		log.Fatal("TLS client key file and cert file should both be present")
+	}
+	_, err = exp.CreateClientTLSConfig()
+	if err != nil {
+		log.Fatal(err)
+	}
+
 	log.Infof("Providing metrics at %s%s", *listenAddress, *metricPath)
 	log.Debugf("Configured redis addr: %#v", *redisAddr)
 	if *tlsServerCertFile != "" && *tlsServerKeyFile != "" {
 		log.Debugf("Bind as TLS using cert %s and key %s", *tlsServerCertFile, *tlsServerKeyFile)
-		log.Fatal(http.ListenAndServeTLS(*listenAddress, *tlsServerCertFile, *tlsServerKeyFile, exp))
+
+		// Verify that the initial key pair is accepted
+		_, err := exporter.LoadKeyPair(*tlsServerCertFile, *tlsServerKeyFile)
+		if err != nil {
+			log.Fatalf("Couldn't load TLS server key pair, err: %s", err)
+		}
+		server := &http.Server{
+			Addr:      *listenAddress,
+			TLSConfig: &tls.Config{GetCertificate: exporter.GetServerCertificateFunc(*tlsServerCertFile, *tlsServerKeyFile)},
+			Handler:   exp}
+		log.Fatal(server.ListenAndServeTLS("", ""))
 	} else {
 		log.Fatal(http.ListenAndServe(*listenAddress, exp))
 	}